feat(api): on templated job add the reference of the template use

engine/api/v2_entities.go

@@ -70,7 +70,7 @@ func (api *API) postEntityCheckHandler() ([]service.RbacChecker, service.Handler
 				response.Messages = append(response.Messages, fmt.Sprintf("%q", err))
 			}
 			if err == nil {
-				errs := wm.Lint()
+				errs := wm.LintYamlDefinition()
 				for _, err := range errs {
 					response.Messages = append(response.Messages, err.Error())
 				}
@@ -82,7 +82,7 @@ func (api *API) postEntityCheckHandler() ([]service.RbacChecker, service.Handler
 				response.Messages = append(response.Messages, fmt.Sprintf("%q", err))
 			}
 			if err == nil {
-				errs := a.Lint()
+				errs := a.LintYamlDefinition()
 				for _, err := range errs {
 					response.Messages = append(response.Messages, err.Error())
 				}
@@ -94,7 +94,7 @@ func (api *API) postEntityCheckHandler() ([]service.RbacChecker, service.Handler
 				response.Messages = append(response.Messages, fmt.Sprintf("%q", err))
 			}
 			if err == nil {
-				errs := w.Lint()
+				errs := w.LintYamlDefinition()
 				for _, err := range errs {
 					response.Messages = append(response.Messages, err.Error())
 				}
@@ -106,7 +106,7 @@ func (api *API) postEntityCheckHandler() ([]service.RbacChecker, service.Handler
 				response.Messages = append(response.Messages, fmt.Sprintf("%q", err))
 			}
 			if err == nil {
-				errs := wt.Lint()
+				errs := wt.LintYamlDefinition()
 				for _, err := range errs {
 					response.Messages = append(response.Messages, err.Error())
 				}

engine/api/v2_repository_analyze.go

@@ -1656,7 +1656,7 @@ func (api *API) handleEntitiesFiles(ctx context.Context, ef *EntityFinder, files
 // and concurrencies belong to the run's project.
 func Lint[T sdk.Lintable](ctx context.Context, db *gorp.DbMap, store cache.Store, o T, ef *EntityFinder, ownerProjectKey string, wmDockerImageWhiteList []regexp.Regexp) []error {
 	// 1. Static lint
-	if err := o.Lint(); err != nil {
+	if err := o.LintYamlDefinition(); err != nil {
 		return err
 	}
 

engine/api/v2_workflow_run_craft.go

@@ -316,7 +316,7 @@ func (api *API) craftWorkflowRunV2(ctx context.Context, id string) error {
 
 	// Check workflow lint in case of modification through job template
 	msgs := make([]sdk.V2WorkflowRunInfo, 0)
-	errs := run.WorkflowData.Workflow.Lint()
+	errs := run.WorkflowData.Workflow.LintYamlDefinition()
 	for _, e := range errs {
 		msgs = append(msgs, sdk.V2WorkflowRunInfo{
 			WorkflowRunID: run.ID,
@@ -558,7 +558,7 @@ func (api *API) craftWorkflowRunV2(ctx context.Context, id string) error {
 }
 
 func retrieveAndUpdateAllJobDependencies(ctx context.Context, db *gorp.DbMap, store cache.Store, run *sdk.V2WorkflowRun, jobID string, j sdk.V2Job, wref *WorkflowRunEntityFinder, integrations map[string]sdk.ProjectIntegration, allVariableSets []sdk.ProjectVariableSet, defaultRegion string) *sdk.V2WorkflowRunInfo {
-	if len(j.Steps) == 0 && j.From == "" {
+	if len(j.Steps) == 0 && !j.NeedsTemplateResolution() {
 		return nil
 	}
 
@@ -598,7 +598,7 @@ func retrieveAndUpdateAllJobDependencies(ctx context.Context, db *gorp.DbMap, st
 	}
 
 	// Check worker model
-	if !strings.Contains(j.RunsOn.Model, "${{") && j.From == "" && !strings.Contains(j.Region, "${{") {
+	if !strings.Contains(j.RunsOn.Model, "${{") && !j.NeedsTemplateResolution() && !strings.Contains(j.Region, "${{") {
 		completeName, msg, err := wref.checkWorkerModel(ctx, db, store, jobID, j.RunsOn.Model, j.Region, defaultRegion)
 		if err != nil {
 			log.ErrorWithStackTrace(ctx, err)

engine/api/v2_workflow_run_craft_test.go

@@ -3555,3 +3555,152 @@ spec: |-
 	require.Equal(t, 0, len(wrInfos), "Error found: %v", wrInfos)
 	require.Equal(t, sdk.V2WorkflowRunStatusBuilding, wrDB.Status)
 }
+
+// A workflow holding a job template reference: crafting must leave the reference
+// untouched (no worker model resolution on it) while resolving the dependencies of
+// the sibling concrete job.
+func TestCraftWorkflowRunWithJobTemplateReference(t *testing.T) {
+	api, db, _ := newTestAPI(t)
+	ctx := context.TODO()
+
+	db.Exec("DELETE FROM rbac")
+	db.Exec("DELETE FROM region")
+
+	reg := sdk.Region{Name: "build"}
+	require.NoError(t, region.Insert(ctx, db, &reg))
+	api.Config.Workflow.JobDefaultRegion = reg.Name
+
+	proj := assets.InsertTestProject(t, db, api.Cache, sdk.RandomString(10), sdk.RandomString(10))
+	admin, _ := assets.InsertAdminUser(t, db)
+
+	vcsProject := assets.InsertTestVCSProject(t, db, proj.ID, "github", "github")
+	repo := assets.InsertTestProjectRepository(t, db, proj.Key, vcsProject.ID, sdk.RandomString(10))
+
+	s, _ := assets.InsertService(t, db, t.Name()+"_VCS", sdk.TypeVCS)
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+	servicesClients := mock_services.NewMockClient(ctrl)
+	services.NewClient = func(_ []sdk.Service) services.Client {
+		return servicesClients
+	}
+	t.Cleanup(func() {
+		_ = services.Delete(db, s)
+		services.NewClient = services.NewDefaultClient
+	})
+	servicesClients.EXPECT().
+		DoJSONRequest(gomock.Any(), "GET", "/vcs/github/repos/"+repo.Name, gomock.Any(), gomock.Any(), gomock.Any()).
+		DoAndReturn(
+			func(ctx context.Context, method, path string, in interface{}, out interface{}, _ interface{}) (http.Header, int, error) {
+				b := &sdk.VCSRepo{}
+				*(out.(*sdk.VCSRepo)) = *b
+				return nil, 200, nil
+			},
+		).Times(1)
+
+	entityTmpl := sdk.Entity{
+		ProjectKey:          proj.Key,
+		ProjectRepositoryID: repo.ID,
+		Type:                sdk.EntityTypeWorkflowTemplate,
+		FilePath:            ".cds/workflow-templates/mytmpl.yml",
+		Name:                "myTemplate",
+		Ref:                 "refs/heads/master",
+		Commit:              "123456789",
+		Data: `name: mytemplate
+spec: |-
+  jobs:
+    fromTemplate:
+      runs-on: .cds/worker-models/myworker-model.yml
+      steps:
+      - run: echo "from template"`,
+	}
+	require.NoError(t, entity.Insert(ctx, db, &entityTmpl))
+
+	myWMEnt := sdk.Entity{
+		ProjectKey:          proj.Key,
+		ProjectRepositoryID: repo.ID,
+		Type:                sdk.EntityTypeWorkerModel,
+		FilePath:            ".cds/worker-models/myworker-model.yml",
+		Name:                "myworker-model",
+		Ref:                 "refs/heads/master",
+		Commit:              "123456789",
+		Data:                "name: myworkermodel",
+	}
+	require.NoError(t, entity.Insert(ctx, db, &myWMEnt))
+
+	hatch := sdk.Hatchery{Name: sdk.RandomString(10), ModelType: ""}
+	require.NoError(t, hatchery.Insert(ctx, db, &hatch))
+	perm := sdk.RBAC{
+		Name: sdk.RandomString(10),
+		Hatcheries: []sdk.RBACHatchery{
+			{
+				RegionID:   reg.ID,
+				HatcheryID: hatch.ID,
+				Role:       sdk.HatcheryRoleSpawn,
+			},
+		},
+	}
+	require.NoError(t, rbac.Insert(ctx, db, &perm))
+
+	wkName := sdk.RandomString(10)
+	wr := sdk.V2WorkflowRun{
+		DeprecatedUserID: admin.ID,
+		ProjectKey:       proj.Key,
+		Status:           sdk.V2WorkflowRunStatusCrafting,
+		VCSServerID:      vcsProject.ID,
+		RepositoryID:     repo.ID,
+		RunNumber:        0,
+		RunAttempt:       0,
+		WorkflowRef:      "refs/heads/master",
+		WorkflowSha:      "123456789",
+		WorkflowName:     wkName,
+		WorkflowData: sdk.V2WorkflowRunData{
+			Workflow: sdk.V2Workflow{
+				Name: wkName,
+				Jobs: map[string]sdk.V2Job{
+					"tmplJob": {
+						From: ".cds/workflow-templates/mytmpl.yml",
+					},
+					"normalJob": {
+						RunsOn: sdk.V2JobRunsOn{
+							Model: "myworker-model",
+						},
+						Steps: []sdk.ActionStep{
+							{ID: "step1", Run: "echo hello"},
+						},
+					},
+				},
+			},
+		},
+		Initiator: &sdk.V2Initiator{
+			UserID:         admin.ID,
+			IsAdminWithMFA: true,
+		},
+		RunEvent: sdk.V2WorkflowRunEvent{
+			HookType:  sdk.WorkflowHookTypeRepository,
+			Payload:   nil,
+			Ref:       "refs/heads/master",
+			Sha:       "123456789",
+			EventName: sdk.WorkflowHookEventNamePush,
+		},
+	}
+	require.NoError(t, workflow_v2.InsertRun(ctx, db, &wr))
+
+	require.NoError(t, api.craftWorkflowRunV2(ctx, wr.ID))
+
+	wrDB, err := workflow_v2.LoadRunByID(ctx, db, wr.ID)
+	require.NoError(t, err)
+	wrInfos, err := workflow_v2.LoadRunInfosByRunID(ctx, db, wr.ID)
+	require.NoError(t, err)
+	require.Equal(t, 0, len(wrInfos), "Error found: %v", wrInfos)
+	require.Equal(t, sdk.V2WorkflowRunStatusBuilding, wrDB.Status)
+
+	// The job template reference is kept as is: no expansion, no model resolution
+	tmplJob := wrDB.WorkflowData.Workflow.Jobs["tmplJob"]
+	require.Equal(t, ".cds/workflow-templates/mytmpl.yml", tmplJob.From)
+	require.Empty(t, tmplJob.RunsOn.Model)
+	require.Empty(t, tmplJob.Steps)
+
+	// The concrete job got its worker model resolved
+	normalJob := wrDB.WorkflowData.Workflow.Jobs["normalJob"]
+	require.Contains(t, normalJob.RunsOn.Model, "myworker-model@refs/heads/master")
+}

engine/api/v2_workflow_run_engine.go

@@ -325,7 +325,7 @@ func (api *API) workflowRunV2Trigger(ctx context.Context, wrEnqueue sdk.V2Workfl
 	// Enqueue JOB
 	hasTemplatedJob := false
 	for _, j := range jobsToQueue {
-		if j.Job.From != "" {
+		if j.Job.NeedsTemplateResolution() {
 			hasTemplatedJob = true
 		}
 	}
@@ -1536,7 +1536,7 @@ func prepareRunJobs(ctx context.Context, db *gorp.DbMap, store cache.Store, proj
 		}
 
 		// No matrixed jobs or templated job skipped
-		if len(matrixPermutation) == 0 || (jobDef.From != "" && jobToTrigger.Status.IsTerminated()) {
+		if len(matrixPermutation) == 0 || (jobDef.NeedsTemplateResolution() && jobToTrigger.Status.IsTerminated()) {
 			runJob := sdk.V2WorkflowRunJob{
 				ID:                 sdk.UUID(),
 				WorkflowRunID:      run.ID,
@@ -1555,7 +1555,7 @@ func prepareRunJobs(ctx context.Context, db *gorp.DbMap, store cache.Store, proj
 				RunAttempt:         run.RunAttempt,
 				Initiator:          wrEnqueue.Initiator,
 			}
-			if jobDef.From == "" && len(jobDef.Steps) == 0 && !jobToTrigger.Status.IsTerminated() {
+			if !jobDef.NeedsTemplateResolution() && len(jobDef.Steps) == 0 && !jobToTrigger.Status.IsTerminated() {
 				runJob.Status = sdk.V2WorkflowRunJobStatusSuccess
 			}
 			// If the current job was a matrix, skip it
@@ -1572,7 +1572,7 @@ func prepareRunJobs(ctx context.Context, db *gorp.DbMap, store cache.Store, proj
 			// If job has to be run
 			if !runJob.Status.IsTerminated() {
 				// If from template, retrieve template and apply it
-				if jobDef.From != "" {
+				if jobDef.NeedsTemplateResolution() {
 					hasToUpdateRun = true
 					// For templated job, we only create new jobs on the parent worklow
 					// With hasToUpdateRun the workflow run will be saved to update his definition
@@ -1660,7 +1660,7 @@ func prepareRunJobs(ctx context.Context, db *gorp.DbMap, store cache.Store, proj
 				allVariableSets: allVariableSets,
 			}
 
-			if jobDef.From == "" {
+			if !jobDef.NeedsTemplateResolution() {
 				jobs, runUpdated, err := createMatrixedRunJobs(ctx, db, store, wref, matrixPermutation, runJobsInfo, run, jobData, concurrenciesDef, concurrencyUnlockedCount, runObjectsToCancelled)
 				if err != nil {
 					return nil, nil, nil, nil, false, err
@@ -1791,7 +1791,7 @@ func computeJobFromTemplate(ctx context.Context, db *gorp.DbMap, store cache.Sto
 	}
 
 	msgsLint := make([]sdk.V2WorkflowRunInfo, 0)
-	errs := run.WorkflowData.Workflow.Lint()
+	errs := run.WorkflowData.Workflow.LintWorkflowRunData()
 	for _, e := range errs {
 		msgsLint = append(msgsLint, sdk.V2WorkflowRunInfo{
 			WorkflowRunID: run.ID,
@@ -1892,6 +1892,12 @@ loop:
 
 	// Set job on workflow
 	for k, v := range newJobs {
+		// Jobs resolved from the template keep the template's complete name as
+		// provenance. Empty jobs are skipped: `from` without content would read as
+		// an unresolved reference. Nested references keep their own `from`.
+		if v.From == "" && (len(v.Steps) > 0 || v.RunsOn.Model != "") {
+			v.From = templateEntity.CompleteName
+		}
 		run.WorkflowData.Workflow.Jobs[k] = v
 		msg := retrieveAndUpdateAllJobDependencies(ctx, db, store, run, k, v, wrefTemplate, integrations, allVariableSets, defaultRegion)
 		if msg != nil {
@@ -2102,7 +2108,7 @@ func createTemplatedMatrixedJobs(ctx context.Context, db *gorp.DbMap, store cach
 	}
 
 	msgsLint := make([]sdk.V2WorkflowRunInfo, 0)
-	errs := run.WorkflowData.Workflow.Lint()
+	errs := run.WorkflowData.Workflow.LintWorkflowRunData()
 	for _, e := range errs {
 		msgsLint = append(msgsLint, sdk.V2WorkflowRunInfo{
 			WorkflowRunID: run.ID,
@@ -2373,7 +2379,7 @@ func retrieveJobToQueue(ctx context.Context, db *gorp.DbMap, wrEnqueue sdk.V2Wor
 			if runJobMapItem.Job.Strategy != nil && len(runJobMapItem.Job.Strategy.Matrix) > 0 {
 
 				// If runjob has a status && a template, ignore it. A matrix job can be run if template has been resolved
-				if runJobMapItem.Job.From != "" {
+				if runJobMapItem.Job.NeedsTemplateResolution() {
 					continue
 				}