diff --git a/pkg/microservice/aslan/core/common/repository/models/wokflow_task_v4.go b/pkg/microservice/aslan/core/common/repository/models/wokflow_task_v4.go index 1988627e8d..4eb08a3705 100644 --- a/pkg/microservice/aslan/core/common/repository/models/wokflow_task_v4.go +++ b/pkg/microservice/aslan/core/common/repository/models/wokflow_task_v4.go @@ -303,6 +303,8 @@ type JobTaskHelmDeploySpec struct { KeyVals []*ServiceKeyVal `bson:"key_vals" json:"key_vals" yaml:"key_vals"` // VariableYaml stores the variable YAML provided by user VariableYaml string `bson:"variable_yaml" json:"variable_yaml" yaml:"variable_yaml"` + // OverrideKVs stores Helm kv values as a JSON-encoded key-value list. + OverrideKVs string `bson:"override_kvs" json:"override_kvs" yaml:"override_kvs"` // IsProduction added since 1.18, indicator of production environment deployment job IsProduction bool `bson:"is_production" yaml:"is_production" json:"is_production"` YamlContent string `bson:"yaml_content" json:"yaml_content" yaml:"yaml_content"` diff --git a/pkg/microservice/aslan/core/common/service/helm/util.go b/pkg/microservice/aslan/core/common/service/helm/util.go index 5b4e5aa35d..4b7d27550e 100644 --- a/pkg/microservice/aslan/core/common/service/helm/util.go +++ b/pkg/microservice/aslan/core/common/service/helm/util.go @@ -16,7 +16,12 @@ limitations under the License. package helm -import "gopkg.in/yaml.v3" +import ( + "encoding/json" + + helmtool "github.com/koderover/zadig/v2/pkg/tool/helmclient" + "gopkg.in/yaml.v3" +) func GetValuesMapFromString(values string) (map[string]interface{}, error) { ret := make(map[string]interface{}) @@ -47,3 +52,56 @@ func MergeHelmValues(oldVals, newVals map[string]interface{}) map[string]interfa } return result } + +// MergeOverrideKVStrings merges Helm --set values by key. Values in overlay +// take precedence over values in base. This is used when a request only +// contains user-editable values: environment-persisted values must remain in +// the result when they are omitted from the request. +func MergeOverrideKVStrings(base, overlay string) (string, error) { + merged := make([]*helmtool.KV, 0) + upsert := func(kv *helmtool.KV) { + if kv == nil || kv.Key == "" { + return + } + for i := len(merged) - 1; i >= 0; i-- { + if merged[i].Key == kv.Key { + merged[i].Value = kv.Value + return + } + } + merged = append(merged, kv) + } + decode := func(values string) error { + kvs := make([]*helmtool.KV, 0) + if err := json.Unmarshal([]byte(values), &kvs); err != nil { + return err + } + for _, kv := range kvs { + upsert(kv) + } + return nil + } + + if base != "" { + if err := decode(base); err != nil { + return "", err + } + } + if overlay != "" { + if err := decode(overlay); err != nil { + return "", err + } + } + if len(merged) == 0 { + if overlay != "" { + return overlay, nil + } + return base, nil + } + + result, err := json.Marshal(merged) + if err != nil { + return "", err + } + return string(result), nil +} diff --git a/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job_helm_deploy.go b/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job_helm_deploy.go index 38cd9b9194..ce167a939c 100644 --- a/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job_helm_deploy.go +++ b/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job_helm_deploy.go @@ -18,7 +18,9 @@ package jobcontroller import ( "context" + "encoding/json" "fmt" + "regexp" "strings" "time" @@ -47,6 +49,8 @@ import ( "github.com/koderover/zadig/v2/pkg/util" ) +var helmVariableRegexp = regexp.MustCompile(config.VariableRegEx) + type HelmDeployJobCtl struct { job *commonmodels.JobTask namespace string @@ -150,6 +154,15 @@ func (c *HelmDeployJobCtl) Run(ctx context.Context) { } newEnvService.DeployStrategy = setting.ServiceDeployStrategyDeploy productInfo.ServiceDeployStrategy[c.jobTaskSpec.ServiceName] = setting.ServiceDeployStrategyDeploy + if slices.Contains(c.jobTaskSpec.DeployContents, config.DeployVars) && c.jobTaskSpec.OverrideKVs != "" { + overrideKVs, err := renderHelmOverrideKVs(c.jobTaskSpec.OverrideKVs, c.workflowCtx.GlobalContextGetAll()) + if err != nil { + logError(c.job, err.Error(), c.logger) + return + } + c.jobTaskSpec.OverrideKVs = overrideKVs + newEnvService.GetServiceRender().OverrideValues = overrideKVs + } // calc final values yaml finalValuesYaml := "" @@ -530,3 +543,37 @@ func (c *HelmDeployJobCtl) getVarsYaml() (string, error) { } return kube.GenerateYamlFromKV(vars) } + +func renderHelmOverrideKVs(origin string, variables map[string]string) (string, error) { + var kvs []*helmtool.KV + if err := json.Unmarshal([]byte(origin), &kvs); err != nil { + return "", fmt.Errorf("failed to decode helm override values: %w", err) + } + + for _, kv := range kvs { + value, ok := kv.Value.(string) + if !ok { + continue + } + unresolved := "" + rendered := helmVariableRegexp.ReplaceAllStringFunc(value, func(ref string) string { + key := strings.TrimSuffix(strings.TrimPrefix(ref, "{{."), "}}") + resolved, ok := variables[key] + if !ok { + unresolved = ref + return ref + } + return resolved + }) + if unresolved != "" { + return "", fmt.Errorf("unresolved helm override variable %s", unresolved) + } + kv.Value = rendered + } + + rendered, err := json.Marshal(kvs) + if err != nil { + return "", fmt.Errorf("failed to encode helm override values: %w", err) + } + return string(rendered), nil +} diff --git a/pkg/microservice/aslan/core/environment/service/environment.go b/pkg/microservice/aslan/core/environment/service/environment.go index b5867d78b9..6025d81809 100644 --- a/pkg/microservice/aslan/core/environment/service/environment.go +++ b/pkg/microservice/aslan/core/environment/service/environment.go @@ -1582,6 +1582,67 @@ type GetHelmValuesDifferenceResp struct { LatestManifestFiles []*kube.HelmManifestFile `json:"latest_manifest_files"` } +// mergeEstimatedOverrideValues keeps environment values, overlays request +// values, and finally applies source=other expressions for the target service. +// Expressions are intentionally left unresolved because this endpoint is also +// used before a workflow has produced its runtime context. +func mergeEstimatedOverrideValues(environmentOrigin string, requestValues []*commonservice.KVPair, serviceConfigs []*commonmodels.DeployServiceVariableConfig, serviceName string) (string, error) { + requestOrigin := "" + if len(requestValues) > 0 { + data, err := json.Marshal(requestValues) + if err != nil { + return "", err + } + requestOrigin = string(data) + } + + mergedOrigin, err := helmservice.MergeOverrideKVStrings(environmentOrigin, requestOrigin) + if err != nil { + return "", err + } + + var merged []*commonservice.KVPair + if mergedOrigin != "" { + if err := json.Unmarshal([]byte(mergedOrigin), &merged); err != nil { + return "", err + } + } + upsert := func(kv *commonservice.KVPair) { + if kv == nil || kv.Key == "" { + return + } + for i := len(merged) - 1; i >= 0; i-- { + if merged[i].Key == kv.Key { + merged[i].Value = kv.Value + return + } + } + merged = append(merged, kv) + } + + changed := false + for _, serviceConfig := range serviceConfigs { + if serviceConfig == nil || serviceConfig.ServiceName != serviceName { + continue + } + for _, config := range serviceConfig.VariableConfigs { + if config != nil && config.Source == "other" && config.VariableKey != "" { + upsert(&commonservice.KVPair{Key: config.VariableKey, Value: config.Value}) + changed = true + } + } + } + if !changed { + return mergedOrigin, nil + } + + data, err := json.Marshal(merged) + if err != nil { + return "", err + } + return string(data), nil +} + func GenEstimatedValues(projectName, envName, namespace, serviceOrReleaseName string, scene EstimateValuesScene, contextType EstimateContentType, format EstimateValuesResponseFormat, arg *EstimateValuesArg, updateServiceRevision, isProduction, isHelmChartDeploy bool, valueMergeStrategy config.ValueMergeStrategy, log *zap.SugaredLogger) (*GetHelmValuesDifferenceResp, error) { var ( prodSvc *commonmodels.ProductService @@ -1739,9 +1800,12 @@ func GenEstimatedValues(projectName, envName, namespace, serviceOrReleaseName st return nil, fmt.Errorf("failed to get chart values, chartRepo: %s, chartName: %s, chartVersion: %s, err %s", arg.ChartRepo, arg.ChartName, arg.ChartVersion, err) } - tempArg := &commonservice.HelmSvcRenderArg{OverrideValues: arg.OverrideValues} + overrideValues, err := mergeEstimatedOverrideValues(prodSvc.GetServiceRender().OverrideValues, arg.OverrideValues, arg.ServiceVariableConfig, serviceOrReleaseName) + if err != nil { + return nil, fmt.Errorf("failed to merge environment override values, err: %s", err) + } prodSvc.GetServiceRender().SetOverrideYaml(arg.OverrideYaml) - prodSvc.GetServiceRender().OverrideValues = tempArg.ToOverrideValueString() + prodSvc.GetServiceRender().OverrideValues = overrideValues helmDeploySvc := helmservice.NewHelmDeployService() mergedYaml, err := helmDeploySvc.GenMergedValues(prodSvc, prod.DefaultValues, nil) @@ -1778,7 +1842,10 @@ func GenEstimatedValues(projectName, envName, namespace, serviceOrReleaseName st } else { latestReleaseName = util.GeneReleaseName(latestTmplSvc.GetReleaseNaming(), projectName, prod.Namespace, envName, latestTmplSvc.ServiceName) - tempArg := &commonservice.HelmSvcRenderArg{OverrideValues: arg.OverrideValues} + overrideValues, err := mergeEstimatedOverrideValues(prodSvc.GetServiceRender().OverrideValues, arg.OverrideValues, arg.ServiceVariableConfig, serviceOrReleaseName) + if err != nil { + return nil, fmt.Errorf("failed to merge environment override values, err: %s", err) + } overrideValue := arg.OverrideYaml if valueMergeStrategy == config.ValueMergeStrategyReuseValue { envValuesMap, err := helmservice.GetValuesMapFromString(prodSvc.GetServiceRender().GetOverrideYaml()) @@ -1800,7 +1867,7 @@ func GenEstimatedValues(projectName, envName, namespace, serviceOrReleaseName st } prodSvc.GetServiceRender().SetOverrideYaml(overrideValue) - prodSvc.GetServiceRender().OverrideValues = tempArg.ToOverrideValueString() + prodSvc.GetServiceRender().OverrideValues = overrideValues helmDeploySvc := helmservice.NewHelmDeployService() yamlContent, err := helmDeploySvc.GenMergedValues(prodSvc, prod.DefaultValues, nil) diff --git a/pkg/microservice/aslan/core/environment/service/environment_define.go b/pkg/microservice/aslan/core/environment/service/environment_define.go index 3d4686384c..ed154406c9 100644 --- a/pkg/microservice/aslan/core/environment/service/environment_define.go +++ b/pkg/microservice/aslan/core/environment/service/environment_define.go @@ -126,13 +126,14 @@ type ProductParams struct { } type EstimateValuesArg struct { - ChartRepo string `json:"chartRepo,omitempty"` - ChartName string `json:"chartName,omitempty"` - ChartVersion string `json:"chartVersion,omitempty"` - DefaultValues string `json:"defaultValues"` - OverrideYaml string `json:"overrideYaml"` - OverrideValues []*commonservice.KVPair `json:"overrideValues,omitempty"` - Production bool `json:"-"` + ChartRepo string `json:"chartRepo,omitempty"` + ChartName string `json:"chartName,omitempty"` + ChartVersion string `json:"chartVersion,omitempty"` + DefaultValues string `json:"defaultValues"` + OverrideYaml string `json:"overrideYaml"` + OverrideValues []*commonservice.KVPair `json:"overrideValues,omitempty"` + ServiceVariableConfig []*commonmodels.DeployServiceVariableConfig `json:"serviceVariableConfig,omitempty" yaml:"service_variable_config,omitempty"` + Production bool `json:"-"` } type EnvRenderChartArg struct { diff --git a/pkg/microservice/aslan/core/workflow/handler/helm_variable_fields.go b/pkg/microservice/aslan/core/workflow/handler/helm_variable_fields.go new file mode 100644 index 0000000000..b4570067b2 --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/handler/helm_variable_fields.go @@ -0,0 +1,28 @@ +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/workflow/service/workflow" + internalhandler "github.com/koderover/zadig/v2/pkg/shared/handler" + e "github.com/koderover/zadig/v2/pkg/tool/errors" +) + +// GetHelmVariableFields returns the latest flat Helm values for a service. +// The service is read from the project template, so it does not need to exist +// in the selected environment yet. +func GetHelmVariableFields(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + projectName := c.Query("projectName") + serviceName := c.Query("serviceName") + if projectName == "" || serviceName == "" { + ctx.RespErr = e.ErrInvalidParam.AddDesc("projectName and serviceName are required") + return + } + + production := c.Query("production") == "true" + envName := c.Query("envName") + ctx.Resp, ctx.RespErr = workflow.GetHelmVariableFields(projectName, serviceName, envName, production, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/workflow/handler/router.go b/pkg/microservice/aslan/core/workflow/handler/router.go index bef1d40a4c..02c17f2d04 100644 --- a/pkg/microservice/aslan/core/workflow/handler/router.go +++ b/pkg/microservice/aslan/core/workflow/handler/router.go @@ -80,6 +80,7 @@ func (*Router) Inject(router *gin.RouterGroup) { workflowV4.POST("/:name/template-binding/resolve", ResolveWorkflowTemplateBinding) workflowV4.POST("/output/:jobName", GetWorkflowGlobalVars) workflowV4.POST("/repo/:jobName", GetWorkflowRepoIndex) + workflowV4.POST("/deploy/helm/variable-fields", GetHelmVariableFields) workflowV4.GET("/name/:name", FindWorkflowV4) workflowV4.PUT("/:name", UpdateWorkflowV4) workflowV4.DELETE("/:name", DeleteWorkflowV4) diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/controller/job/job_deploy.go b/pkg/microservice/aslan/core/workflow/service/workflow/controller/job/job_deploy.go index e5317c98b4..fbe1ce346d 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/controller/job/job_deploy.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/controller/job/job_deploy.go @@ -18,6 +18,7 @@ package job import ( "context" + "encoding/json" "fmt" "strings" "time" @@ -764,6 +765,11 @@ func (j DeployJobController) ToTask(taskID int64) ([]*commonmodels.JobTask, erro } if j.jobSpec.DeployType == setting.HelmDeployType { + serviceVariableConfigMap := make(map[string][]*commonmodels.DeployVariableConfig) + for _, serviceConfig := range j.jobSpec.ServiceVariableConfig { + serviceVariableConfigMap[serviceConfig.ServiceName] = serviceConfig.VariableConfigs + } + for jobSubTaskID, svc := range j.jobSpec.Services { var serviceRevision int64 var autoSyncFlag bool @@ -810,6 +816,24 @@ func (j DeployJobController) ToTask(taskID int64) ([]*commonmodels.JobTask, erro ValueMergeStrategy: svc.ValueMergeStrategy, MaxHistory: templateProduct.ReleaseMaxHistory, } + if slices.Contains(j.jobSpec.DeployContents, config.DeployVars) { + // The environment render is the source of truth for existing Helm + // override values. The workflow payload may omit values that are not + // exposed as configurable variables, so merge it on top of the + // environment values before applying `other` references. + envOverrideKVs := "" + if envSvc := productServiceMap[svc.ServiceName]; envSvc != nil { + envOverrideKVs = envSvc.GetServiceRender().OverrideValues + } + jobTaskSpec.OverrideKVs, err = mergeHelmOverrideKVsWithEnvironment( + envOverrideKVs, + svc.OverrideKVs, + serviceVariableConfigMap[svc.ServiceName], + ) + if err != nil { + return nil, fmt.Errorf("failed to merge override values for service %s: %w", svc.ServiceName, err) + } + } for _, module := range svc.Modules { service := serviceMap[svc.ServiceName] @@ -1419,6 +1443,67 @@ func filterKVsByConfig(serviceName string, originKVs []*commontypes.RenderVariab return resp } +func mergeHelmOverrideKVs(origin string, configs []*commonmodels.DeployVariableConfig) (string, error) { + return mergeHelmOverrideKVsWithEnvironment("", origin, configs) +} + +// mergeHelmOverrideKVsWithEnvironment builds the task override values in +// precedence order: environment values, workflow-supplied values, and then +// source=other variable references. This keeps environment-only keys when the +// frontend does not include them in the workflow payload. +func mergeHelmOverrideKVsWithEnvironment(environmentOrigin, taskOrigin string, configs []*commonmodels.DeployVariableConfig) (string, error) { + kvs := make([]*commonservice.KVPair, 0) + upsert := func(kv *commonservice.KVPair) { + for i := len(kvs) - 1; i >= 0; i-- { + if kvs[i].Key == kv.Key { + kvs[i].Value = kv.Value + return + } + } + kvs = append(kvs, kv) + } + decodeAndUpsert := func(origin string) error { + if origin == "" { + return nil + } + originKVs := make([]*commonservice.KVPair, 0) + if err := json.Unmarshal([]byte(origin), &originKVs); err != nil { + return err + } + for _, kv := range originKVs { + if kv != nil && kv.Key != "" { + upsert(kv) + } + } + return nil + } + + if err := decodeAndUpsert(environmentOrigin); err != nil { + return "", err + } + if err := decodeAndUpsert(taskOrigin); err != nil { + return "", err + } + + for _, config := range configs { + if config.Source != "other" { + continue + } + if config.VariableKey != "" { + upsert(&commonservice.KVPair{Key: config.VariableKey, Value: config.Value}) + } + } + + if len(kvs) == 0 && environmentOrigin == "" && taskOrigin == "" { + return "", nil + } + data, err := json.Marshal(kvs) + if err != nil { + return "", err + } + return string(data), nil +} + func checkServiceExistsInEnv(serviceMap map[string]*commonmodels.ProductService, serviceName, env string) error { if _, ok := serviceMap[serviceName]; !ok { return fmt.Errorf("service %s not exists in env %s", serviceName, env) diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/helm_variable_fields.go b/pkg/microservice/aslan/core/workflow/service/workflow/helm_variable_fields.go new file mode 100644 index 0000000000..72af9a0c90 --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/service/workflow/helm_variable_fields.go @@ -0,0 +1,78 @@ +package workflow + +import ( + "fmt" + + "github.com/pkg/errors" + "go.uber.org/zap" + + commonrepo "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/mongodb" + servicerepo "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/service/repository" + "github.com/koderover/zadig/v2/pkg/setting" + "github.com/koderover/zadig/v2/pkg/tool/helmclient" + "github.com/koderover/zadig/v2/pkg/util/converter" +) + +type HelmVariableFieldsResponse struct { + LatestFlatMap map[string]interface{} `json:"latest_flat_map"` +} + +// GetHelmVariableFields resolves template values and optional environment +// values. The template is the source of truth when the service has not been +// created in the environment yet. +func GetHelmVariableFields(projectName, serviceName, envName string, production bool, logger *zap.SugaredLogger) (*HelmVariableFieldsResponse, error) { + templateService, err := servicerepo.QueryTemplateService(&commonrepo.ServiceFindOption{ + ServiceName: serviceName, + ProductName: projectName, + Type: setting.HelmDeployType, + }, production) + if err != nil { + return nil, errors.Wrapf(err, "failed to find helm service %s in project %s", serviceName, projectName) + } + if templateService.HelmChart == nil { + return nil, fmt.Errorf("service %s in project %s is not a helm service", serviceName, projectName) + } + + valuesYAML := templateService.HelmChart.ValuesYaml + if envName != "" { + productionValue := production + env, findErr := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: projectName, + EnvName: envName, + Production: &productionValue, + }) + if findErr != nil { + return nil, errors.Wrapf(findErr, "failed to find environment %s", envName) + } + + var envYAML, envOverrideValues string + if envService := env.GetServiceMap()[serviceName]; envService != nil { + render := envService.GetServiceRender() + envYAML = render.GetSafeVariable() + envOverrideValues = render.OverrideValues + } + valuesYAML, err = helmclient.MergeOverrideValues(valuesYAML, env.DefaultValues, envYAML, envOverrideValues, nil) + if err != nil { + return nil, errors.Wrap(err, "failed to merge environment helm values") + } + } + + response, err := helmVariableFlatMapFromYAML(valuesYAML) + if err != nil { + return nil, errors.Wrap(err, "failed to parse helm values") + } + if logger != nil { + logger.Debugw("resolved helm variable fields", "projectName", projectName, "serviceName", serviceName, "envName", envName) + } + return response, nil +} + +// helmVariableFlatMapFromYAML is kept independent from MongoDB so the value +// flattening can be tested without external dependencies. +func helmVariableFlatMapFromYAML(valuesYAML string) (*HelmVariableFieldsResponse, error) { + flatMap, err := converter.YamlToFlatMap([]byte(valuesYAML)) + if err != nil { + return nil, err + } + return &HelmVariableFieldsResponse{LatestFlatMap: flatMap}, nil +} diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/workflow_task_v4.go b/pkg/microservice/aslan/core/workflow/service/workflow/workflow_task_v4.go index 8fdd08fe4b..3665e9503f 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/workflow_task_v4.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/workflow_task_v4.go @@ -192,6 +192,7 @@ type ZadigDeployJobPreviewSpec struct { YamlContent string `bson:"yaml_content" json:"yaml_content"` // UserSuppliedValue added since 1.18, the values that users gives. UserSuppliedValue string `bson:"user_supplied_value" json:"user_supplied_value" yaml:"user_supplied_value"` + OverrideKVs string `bson:"override_kvs" json:"override_kvs" yaml:"override_kvs"` // VariableKVs new since 1.18, only used for k8s VariableKVs []*commontypes.RenderVariableKV `bson:"variable_kvs" json:"variable_kvs" yaml:"variable_kvs"` OriginRevision int64 `bson:"origin_revision" json:"origin_revision" yaml:"origin_revision"` @@ -3126,6 +3127,7 @@ func jobsToJobPreviews(jobs []*commonmodels.JobTask, context map[string]string, spec.DeployContents = taskJobSpec.DeployContents spec.YamlContent = taskJobSpec.YamlContent spec.UserSuppliedValue = taskJobSpec.UserSuppliedValue + spec.OverrideKVs = taskJobSpec.OverrideKVs spec.SkipCheckRunStatus = taskJobSpec.SkipCheckRunStatus spec.ValueMergeStrategy = taskJobSpec.ValueMergeStrategy spec.OriginRevision = taskJobSpec.OriginRevision