feat(workflow): resolve Helm service variables

pkg/microservice/aslan/core/common/repository/models/wokflow_task_v4.go

@@ -303,6 +303,8 @@ type JobTaskHelmDeploySpec struct {
 	KeyVals        []*ServiceKeyVal        `bson:"key_vals"                         json:"key_vals"                            yaml:"key_vals"`
 	// VariableYaml stores the variable YAML provided by user
 	VariableYaml string `bson:"variable_yaml"                    json:"variable_yaml"                       yaml:"variable_yaml"`
+	// OverrideKVs stores Helm kv values as a JSON-encoded key-value list.
+	OverrideKVs string `bson:"override_kvs"                     json:"override_kvs"                        yaml:"override_kvs"`
 	// IsProduction added since 1.18, indicator of production environment deployment job
 	IsProduction bool   `bson:"is_production" yaml:"is_production" json:"is_production"`
 	YamlContent  string `bson:"yaml_content"                     json:"yaml_content"                        yaml:"yaml_content"`

pkg/microservice/aslan/core/common/service/helm/util.go

@@ -16,7 +16,12 @@ limitations under the License.
 
 package helm
 
-import "gopkg.in/yaml.v3"
+import (
+	"encoding/json"
+
+	helmtool "github.com/koderover/zadig/v2/pkg/tool/helmclient"
+	"gopkg.in/yaml.v3"
+)
 
 func GetValuesMapFromString(values string) (map[string]interface{}, error) {
 	ret := make(map[string]interface{})
@@ -47,3 +52,56 @@ func MergeHelmValues(oldVals, newVals map[string]interface{}) map[string]interfa
 	}
 	return result
 }
+
+// MergeOverrideKVStrings merges Helm --set values by key. Values in overlay
+// take precedence over values in base. This is used when a request only
+// contains user-editable values: environment-persisted values must remain in
+// the result when they are omitted from the request.
+func MergeOverrideKVStrings(base, overlay string) (string, error) {
+	merged := make([]*helmtool.KV, 0)
+	upsert := func(kv *helmtool.KV) {
+		if kv == nil || kv.Key == "" {
+			return
+		}
+		for i := len(merged) - 1; i >= 0; i-- {
+			if merged[i].Key == kv.Key {
+				merged[i].Value = kv.Value
+				return
+			}
+		}
+		merged = append(merged, kv)
+	}
+	decode := func(values string) error {
+		kvs := make([]*helmtool.KV, 0)
+		if err := json.Unmarshal([]byte(values), &kvs); err != nil {
+			return err
+		}
+		for _, kv := range kvs {
+			upsert(kv)
+		}
+		return nil
+	}
+
+	if base != "" {
+		if err := decode(base); err != nil {
+			return "", err
+		}
+	}
+	if overlay != "" {
+		if err := decode(overlay); err != nil {
+			return "", err
+		}
+	}
+	if len(merged) == 0 {
+		if overlay != "" {
+			return overlay, nil
+		}
+		return base, nil
+	}
+
+	result, err := json.Marshal(merged)
+	if err != nil {
+		return "", err
+	}
+	return string(result), nil
+}

pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job_helm_deploy.go

@@ -18,7 +18,9 @@ package jobcontroller
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"regexp"
 	"strings"
 	"time"
 
@@ -47,6 +49,8 @@ import (
 	"github.com/koderover/zadig/v2/pkg/util"
 )
 
+var helmVariableRegexp = regexp.MustCompile(config.VariableRegEx)
+
 type HelmDeployJobCtl struct {
 	job         *commonmodels.JobTask
 	namespace   string
@@ -150,6 +154,15 @@ func (c *HelmDeployJobCtl) Run(ctx context.Context) {
 	}
 	newEnvService.DeployStrategy = setting.ServiceDeployStrategyDeploy
 	productInfo.ServiceDeployStrategy[c.jobTaskSpec.ServiceName] = setting.ServiceDeployStrategyDeploy
+	if slices.Contains(c.jobTaskSpec.DeployContents, config.DeployVars) && c.jobTaskSpec.OverrideKVs != "" {
+		overrideKVs, err := renderHelmOverrideKVs(c.jobTaskSpec.OverrideKVs, c.workflowCtx.GlobalContextGetAll())
+		if err != nil {
+			logError(c.job, err.Error(), c.logger)
+			return
+		}
+		c.jobTaskSpec.OverrideKVs = overrideKVs
+		newEnvService.GetServiceRender().OverrideValues = overrideKVs
+	}
 
 	// calc final values yaml
 	finalValuesYaml := ""
@@ -530,3 +543,37 @@ func (c *HelmDeployJobCtl) getVarsYaml() (string, error) {
 	}
 	return kube.GenerateYamlFromKV(vars)
 }
+
+func renderHelmOverrideKVs(origin string, variables map[string]string) (string, error) {
+	var kvs []*helmtool.KV
+	if err := json.Unmarshal([]byte(origin), &kvs); err != nil {
+		return "", fmt.Errorf("failed to decode helm override values: %w", err)
+	}
+
+	for _, kv := range kvs {
+		value, ok := kv.Value.(string)
+		if !ok {
+			continue
+		}
+		unresolved := ""
+		rendered := helmVariableRegexp.ReplaceAllStringFunc(value, func(ref string) string {
+			key := strings.TrimSuffix(strings.TrimPrefix(ref, "{{."), "}}")
+			resolved, ok := variables[key]
+			if !ok {
+				unresolved = ref
+				return ref
+			}
+			return resolved
+		})
+		if unresolved != "" {
+			return "", fmt.Errorf("unresolved helm override variable %s", unresolved)
+		}
+		kv.Value = rendered
+	}
+
+	rendered, err := json.Marshal(kvs)
+	if err != nil {
+		return "", fmt.Errorf("failed to encode helm override values: %w", err)
+	}
+	return string(rendered), nil
+}

pkg/microservice/aslan/core/environment/service/environment.go

@@ -1582,6 +1582,67 @@ type GetHelmValuesDifferenceResp struct {
 	LatestManifestFiles  []*kube.HelmManifestFile `json:"latest_manifest_files"`
 }
 
+// mergeEstimatedOverrideValues keeps environment values, overlays request
+// values, and finally applies source=other expressions for the target service.
+// Expressions are intentionally left unresolved because this endpoint is also
+// used before a workflow has produced its runtime context.
+func mergeEstimatedOverrideValues(environmentOrigin string, requestValues []*commonservice.KVPair, serviceConfigs []*commonmodels.DeployServiceVariableConfig, serviceName string) (string, error) {
+	requestOrigin := ""
+	if len(requestValues) > 0 {
+		data, err := json.Marshal(requestValues)
+		if err != nil {
+			return "", err
+		}
+		requestOrigin = string(data)
+	}
+
+	mergedOrigin, err := helmservice.MergeOverrideKVStrings(environmentOrigin, requestOrigin)
+	if err != nil {
+		return "", err
+	}
+
+	var merged []*commonservice.KVPair
+	if mergedOrigin != "" {
+		if err := json.Unmarshal([]byte(mergedOrigin), &merged); err != nil {
+			return "", err
+		}
+	}
+	upsert := func(kv *commonservice.KVPair) {
+		if kv == nil || kv.Key == "" {
+			return
+		}
+		for i := len(merged) - 1; i >= 0; i-- {
+			if merged[i].Key == kv.Key {
+				merged[i].Value = kv.Value
+				return
+			}
+		}
+		merged = append(merged, kv)
+	}
+
+	changed := false
+	for _, serviceConfig := range serviceConfigs {
+		if serviceConfig == nil || serviceConfig.ServiceName != serviceName {
+			continue
+		}
+		for _, config := range serviceConfig.VariableConfigs {
+			if config != nil && config.Source == "other" && config.VariableKey != "" {
+				upsert(&commonservice.KVPair{Key: config.VariableKey, Value: config.Value})
+				changed = true
+			}
+		}
+	}
+	if !changed {
+		return mergedOrigin, nil
+	}
+
+	data, err := json.Marshal(merged)
+	if err != nil {
+		return "", err
+	}
+	return string(data), nil
+}
+
 func GenEstimatedValues(projectName, envName, namespace, serviceOrReleaseName string, scene EstimateValuesScene, contextType EstimateContentType, format EstimateValuesResponseFormat, arg *EstimateValuesArg, updateServiceRevision, isProduction, isHelmChartDeploy bool, valueMergeStrategy config.ValueMergeStrategy, log *zap.SugaredLogger) (*GetHelmValuesDifferenceResp, error) {
 	var (
 		prodSvc        *commonmodels.ProductService
@@ -1739,9 +1800,12 @@ func GenEstimatedValues(projectName, envName, namespace, serviceOrReleaseName st
 			return nil, fmt.Errorf("failed to get chart values, chartRepo: %s, chartName: %s, chartVersion: %s, err %s", arg.ChartRepo, arg.ChartName, arg.ChartVersion, err)
 		}
 
-		tempArg := &commonservice.HelmSvcRenderArg{OverrideValues: arg.OverrideValues}
+		overrideValues, err := mergeEstimatedOverrideValues(prodSvc.GetServiceRender().OverrideValues, arg.OverrideValues, arg.ServiceVariableConfig, serviceOrReleaseName)
+		if err != nil {
+			return nil, fmt.Errorf("failed to merge environment override values, err: %s", err)
+		}
 		prodSvc.GetServiceRender().SetOverrideYaml(arg.OverrideYaml)
-		prodSvc.GetServiceRender().OverrideValues = tempArg.ToOverrideValueString()
+		prodSvc.GetServiceRender().OverrideValues = overrideValues
 
 		helmDeploySvc := helmservice.NewHelmDeployService()
 		mergedYaml, err := helmDeploySvc.GenMergedValues(prodSvc, prod.DefaultValues, nil)
@@ -1778,7 +1842,10 @@ func GenEstimatedValues(projectName, envName, namespace, serviceOrReleaseName st
 	} else {
 		latestReleaseName = util.GeneReleaseName(latestTmplSvc.GetReleaseNaming(), projectName, prod.Namespace, envName, latestTmplSvc.ServiceName)
 
-		tempArg := &commonservice.HelmSvcRenderArg{OverrideValues: arg.OverrideValues}
+		overrideValues, err := mergeEstimatedOverrideValues(prodSvc.GetServiceRender().OverrideValues, arg.OverrideValues, arg.ServiceVariableConfig, serviceOrReleaseName)
+		if err != nil {
+			return nil, fmt.Errorf("failed to merge environment override values, err: %s", err)
+		}
 		overrideValue := arg.OverrideYaml
 		if valueMergeStrategy == config.ValueMergeStrategyReuseValue {
 			envValuesMap, err := helmservice.GetValuesMapFromString(prodSvc.GetServiceRender().GetOverrideYaml())
@@ -1800,7 +1867,7 @@ func GenEstimatedValues(projectName, envName, namespace, serviceOrReleaseName st
 		}
 
 		prodSvc.GetServiceRender().SetOverrideYaml(overrideValue)
-		prodSvc.GetServiceRender().OverrideValues = tempArg.ToOverrideValueString()
+		prodSvc.GetServiceRender().OverrideValues = overrideValues
 
 		helmDeploySvc := helmservice.NewHelmDeployService()
 		yamlContent, err := helmDeploySvc.GenMergedValues(prodSvc, prod.DefaultValues, nil)

pkg/microservice/aslan/core/environment/service/environment_define.go

@@ -126,13 +126,14 @@ type ProductParams struct {
 }
 
 type EstimateValuesArg struct {
-	ChartRepo      string                  `json:"chartRepo,omitempty"`
-	ChartName      string                  `json:"chartName,omitempty"`
-	ChartVersion   string                  `json:"chartVersion,omitempty"`
-	DefaultValues  string                  `json:"defaultValues"`
-	OverrideYaml   string                  `json:"overrideYaml"`
-	OverrideValues []*commonservice.KVPair `json:"overrideValues,omitempty"`
-	Production     bool                    `json:"-"`
+	ChartRepo             string                                      `json:"chartRepo,omitempty"`
+	ChartName             string                                      `json:"chartName,omitempty"`
+	ChartVersion          string                                      `json:"chartVersion,omitempty"`
+	DefaultValues         string                                      `json:"defaultValues"`
+	OverrideYaml          string                                      `json:"overrideYaml"`
+	OverrideValues        []*commonservice.KVPair                     `json:"overrideValues,omitempty"`
+	ServiceVariableConfig []*commonmodels.DeployServiceVariableConfig `json:"serviceVariableConfig,omitempty" yaml:"service_variable_config,omitempty"`
+	Production            bool                                        `json:"-"`
 }
 
 type EnvRenderChartArg struct {

pkg/microservice/aslan/core/workflow/handler/helm_variable_fields.go

@@ -0,0 +1,28 @@
+package handler
+
+import (
+	"github.com/gin-gonic/gin"
+
+	"github.com/koderover/zadig/v2/pkg/microservice/aslan/core/workflow/service/workflow"
+	internalhandler "github.com/koderover/zadig/v2/pkg/shared/handler"
+	e "github.com/koderover/zadig/v2/pkg/tool/errors"
+)
+
+// GetHelmVariableFields returns the latest flat Helm values for a service.
+// The service is read from the project template, so it does not need to exist
+// in the selected environment yet.
+func GetHelmVariableFields(c *gin.Context) {
+	ctx := internalhandler.NewContext(c)
+	defer func() { internalhandler.JSONResponse(c, ctx) }()
+
+	projectName := c.Query("projectName")
+	serviceName := c.Query("serviceName")
+	if projectName == "" || serviceName == "" {
+		ctx.RespErr = e.ErrInvalidParam.AddDesc("projectName and serviceName are required")
+		return
+	}
+
+	production := c.Query("production") == "true"
+	envName := c.Query("envName")
+	ctx.Resp, ctx.RespErr = workflow.GetHelmVariableFields(projectName, serviceName, envName, production, ctx.Logger)
+}

pkg/microservice/aslan/core/workflow/handler/router.go

@@ -80,6 +80,7 @@ func (*Router) Inject(router *gin.RouterGroup) {
 		workflowV4.POST("/:name/template-binding/resolve", ResolveWorkflowTemplateBinding)
 		workflowV4.POST("/output/:jobName", GetWorkflowGlobalVars)
 		workflowV4.POST("/repo/:jobName", GetWorkflowRepoIndex)
+		workflowV4.POST("/deploy/helm/variable-fields", GetHelmVariableFields)
 		workflowV4.GET("/name/:name", FindWorkflowV4)
 		workflowV4.PUT("/:name", UpdateWorkflowV4)
 		workflowV4.DELETE("/:name", DeleteWorkflowV4)