feat(configuration): Allow environment variable overrides for config

cmd/dependency/dependency.go

@@ -17,6 +17,7 @@
 package dependency
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -79,15 +80,64 @@ func InitCommandAndConfig(cmd *cobra.Command, useConfigFile bool, config any) {
 		// Config for binding env
 		viper.SetEnvPrefix(rootName)
 		viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
+		viper.AutomaticEnv()
 		_ = viper.BindEnv("config")
 
+		// Bind env for keys absent from the config file
+		bindEnvsFromConfig(config)
+
 		// Add common cmds only on root cmd
 		cmd.AddCommand(VersionCmd)
 		cmd.AddCommand(newDocCommand(cmd.Name()))
 		cmd.AddCommand(PluginCmd)
 	}
 }
 
+// bindEnvsFromConfig binds an env for every config key, so AutomaticEnv can
+// override keys that are not present in the config file.
+func bindEnvsFromConfig(config any) {
+	schema := reflect.New(reflect.TypeOf(config).Elem()).Interface()
+	materializeStructPtrs(reflect.ValueOf(schema).Elem())
+
+	b, err := yaml.Marshal(schema)
+	if err != nil {
+		panic(fmt.Errorf("marshal config for env binding: %w", err))
+	}
+
+	keys := viper.New()
+	keys.SetConfigType("yaml")
+	if err := keys.ReadConfig(bytes.NewReader(b)); err != nil {
+		panic(fmt.Errorf("read config for env binding: %w", err))
+	}
+
+	for _, key := range keys.AllKeys() {
+		_ = viper.BindEnv(key)
+	}
+}
+
+// materializeStructPtrs allocates nil struct-pointers so their nested keys serialize.
+func materializeStructPtrs(v reflect.Value) {
+	switch v.Kind() {
+	case reflect.Pointer:
+		if v.Type().Elem().Kind() != reflect.Struct {
+			return
+		}
+		if v.IsNil() {
+			if !v.CanSet() {
+				return
+			}
+			v.Set(reflect.New(v.Type().Elem()))
+		}
+		materializeStructPtrs(v.Elem())
+	case reflect.Struct:
+		for i := 0; i < v.NumField(); i++ {
+			if f := v.Field(i); f.CanSet() {
+				materializeStructPtrs(f)
+			}
+		}
+	}
+}
+
 // InitMonitor initialize monitor and return final handler.
 func InitMonitor(ctx context.Context, pprofPort int, tracingConfig base.TracingConfig) func() {
 	var shutdowns = make(chan func(), 2)

cmd/dependency/dependency_test.go

@@ -0,0 +1,117 @@
+/*
+ *     Copyright 2020 The Dragonfly Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package dependency
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	schedulerconfig "d7y.io/dragonfly/v2/scheduler/config"
+)
+
+// tlsConfig mirrors the optional, pointer-typed TLS sections of the real
+// configs (e.g. scheduler/config.Config.Server.TLS), which are left nil by the
+// default config and therefore never appear in a marshalled value snapshot.
+type tlsConfig struct {
+	CACert string `yaml:"caCert" mapstructure:"caCert"`
+}
+
+// testConfig mirrors the real config shape: populated scalar defaults plus an
+// optional nested section behind a nil pointer.
+type testConfig struct {
+	Name string     `yaml:"name" mapstructure:"name"`
+	TLS  *tlsConfig `yaml:"tls" mapstructure:"tls"`
+}
+
+// newTestConfig returns the default config: scalar defaults set, optional TLS
+// section left nil (as config.New() does for the real configs).
+func newTestConfig() *testConfig {
+	return &testConfig{Name: "default-name"}
+}
+
+// setupViper resets and configures the package-global viper the same way
+// InitCommandAndConfig does, with the given env prefix.
+func setupViper(prefix string) {
+	viper.Reset()
+	viper.SetEnvPrefix(prefix)
+	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
+	viper.AutomaticEnv()
+}
+
+func TestBindEnvsFromConfig_TopLevelOverride(t *testing.T) {
+	setupViper("test")
+	t.Setenv("TEST_NAME", "from-env")
+
+	cfg := newTestConfig()
+	bindEnvsFromConfig(cfg)
+	require.NoError(t, viper.Unmarshal(cfg, initDecoderConfig))
+
+	assert.Equal(t, "from-env", cfg.Name)
+}
+
+// TestBindEnvsFromConfig_NestedNilPointerOverride is the regression guard: the
+// TLS section is nil in the default config, so the previous marshal-of-value
+// implementation never bound TEST_TLS_CACERT and this override was silently
+// dropped. Enumerating keys from the type closes that gap.
+func TestBindEnvsFromConfig_NestedNilPointerOverride(t *testing.T) {
+	setupViper("test")
+	t.Setenv("TEST_TLS_CACERT", "/etc/ssl/ca.crt")
+
+	cfg := newTestConfig()
+	require.Nil(t, cfg.TLS, "TLS should start nil to model the default config")
+
+	bindEnvsFromConfig(cfg)
+	require.NoError(t, viper.Unmarshal(cfg, initDecoderConfig))
+
+	require.NotNil(t, cfg.TLS, "nested env override should materialize the TLS section")
+	assert.Equal(t, "/etc/ssl/ca.crt", cfg.TLS.CACert)
+}
+
+func TestBindEnvsFromConfig_NoEnvKeepsDefaults(t *testing.T) {
+	setupViper("test")
+
+	cfg := newTestConfig()
+	bindEnvsFromConfig(cfg)
+	require.NoError(t, viper.Unmarshal(cfg, initDecoderConfig))
+
+	assert.Equal(t, "default-name", cfg.Name)
+	assert.Nil(t, cfg.TLS, "no env set should leave the optional section nil")
+}
+
+// TestBindEnvsFromConfig_RealSchedulerConfig exercises the actual production
+// config type end-to-end, including a nested key (Server.TLS.CACert) that lives
+// under a pointer left nil by config.New() — the regression the fix targets.
+func TestBindEnvsFromConfig_RealSchedulerConfig(t *testing.T) {
+	setupViper("scheduler")
+	t.Setenv("SCHEDULER_SERVER_HOST", "override-host")
+	t.Setenv("SCHEDULER_SERVER_TLS_CACERT", "/etc/ssl/ca.crt")
+
+	cfg := schedulerconfig.New()
+	require.Nil(t, cfg.Server.TLS, "default scheduler config leaves Server.TLS nil")
+
+	bindEnvsFromConfig(cfg)
+	require.NoError(t, viper.Unmarshal(cfg, initDecoderConfig))
+
+	assert.Equal(t, "override-host", cfg.Server.Host, "env should override a populated default")
+
+	require.NotNil(t, cfg.Server.TLS, "nested env override should materialize the TLS section")
+	assert.Equal(t, "/etc/ssl/ca.crt", cfg.Server.TLS.CACert)
+}

deploy/docker-compose/README.md

@@ -15,6 +15,39 @@ export IP=<host ip>
 ./run.sh
 ```
 
+## Configure with environment variables
+
+In addition to the config files under `config/`, the `manager` and `scheduler`
+services can have any config value overridden with an environment variable. This
+is useful for tweaking a single setting without editing the generated config.
+
+The environment variable name is built from the config key as follows:
+
+- Prefix it with the service name (`MANAGER_` or `SCHEDULER_`).
+- Join nested keys with `_` and uppercase the whole thing.
+- camelCase keys collapse to all-lowercase (no separating underscore), e.g.
+  `advertiseIP` becomes `ADVERTISEIP`.
+
+In other words, the YAML path `a.b.cDef` maps to `<SERVICE>_A_B_CDEF`.
+
+Examples:
+
+| Service   | Config key             | Environment variable            |
+| --------- | ---------------------- | ------------------------------- |
+| scheduler | `server.port`          | `SCHEDULER_SERVER_PORT`         |
+| scheduler | `server.advertiseIP`   | `SCHEDULER_SERVER_ADVERTISEIP`  |
+| manager   | `server.rest.addr`     | `MANAGER_SERVER_REST_ADDR`      |
+| manager   | `server.grpc.port.start` | `MANAGER_SERVER_GRPC_PORT_START` |
+
+For example, to override the scheduler listen port:
+
+```shell
+SCHEDULER_SERVER_PORT=8003 ./run.sh
+```
+
+> Note: this applies to the `manager` and `scheduler` services only. The
+> `client` and `seed-client` services are configured through their config files.
+
 ## Delete containers with docker compose
 
 ```shell