diff --git a/Configuration/PyReleaseValidation/python/relval_2017.py b/Configuration/PyReleaseValidation/python/relval_2017.py index ec6268e223cd7..13ba23b4ff3e0 100644 --- a/Configuration/PyReleaseValidation/python/relval_2017.py +++ b/Configuration/PyReleaseValidation/python/relval_2017.py @@ -1,5 +1,6 @@ # import the definition of the steps and input files: -from Configuration.PyReleaseValidation.relval_steps import * +from Configuration.PyReleaseValidation.relval_steps import * +from .MatrixUtil import Matrix # here only define the workflows as a combination of the steps defined above: workflows = Matrix() diff --git a/Configuration/PyReleaseValidation/python/relval_Run4.py b/Configuration/PyReleaseValidation/python/relval_Run4.py index 994a9a2c9dc15..a8d2e6660ee29 100644 --- a/Configuration/PyReleaseValidation/python/relval_Run4.py +++ b/Configuration/PyReleaseValidation/python/relval_Run4.py @@ -1,5 +1,6 @@ # import the definition of the steps and input files: -from Configuration.PyReleaseValidation.relval_steps import * +from Configuration.PyReleaseValidation.relval_steps import * +from .MatrixUtil import Matrix # here only define the workflows as a combination of the steps defined above: workflows = Matrix() diff --git a/Configuration/PyReleaseValidation/python/relval_cleanedupgrade.py b/Configuration/PyReleaseValidation/python/relval_cleanedupgrade.py index 20ab0f4580a19..31c5829970ff6 100644 --- a/Configuration/PyReleaseValidation/python/relval_cleanedupgrade.py +++ b/Configuration/PyReleaseValidation/python/relval_cleanedupgrade.py @@ -1,5 +1,6 @@ # import the definition of the steps and input files: -from Configuration.PyReleaseValidation.relval_steps import * +from Configuration.PyReleaseValidation.relval_steps import * +from .MatrixUtil import Matrix # here only define the workflows as a combination of the steps defined above: workflows = Matrix() diff --git a/Configuration/PyReleaseValidation/python/relval_data_highstats.py b/Configuration/PyReleaseValidation/python/relval_data_highstats.py index 60d6cce44d34b..63b7d7d4527cc 100644 --- a/Configuration/PyReleaseValidation/python/relval_data_highstats.py +++ b/Configuration/PyReleaseValidation/python/relval_data_highstats.py @@ -1,5 +1,6 @@ # import the definition of the steps and input files: -from Configuration.PyReleaseValidation.relval_steps import * +from Configuration.PyReleaseValidation.relval_steps import * +from .MatrixUtil import Matrix from functools import partial # here only define the workflows as a combination of the steps defined above: diff --git a/Configuration/PyReleaseValidation/python/relval_extendedgen.py b/Configuration/PyReleaseValidation/python/relval_extendedgen.py index 54443eab9e3f3..0b434f7792c69 100644 --- a/Configuration/PyReleaseValidation/python/relval_extendedgen.py +++ b/Configuration/PyReleaseValidation/python/relval_extendedgen.py @@ -2,6 +2,7 @@ # # import the definition of the steps and input files: from Configuration.PyReleaseValidation.relval_steps import * +from .MatrixUtil import Matrix # here only define the workflows as a combination of the steps defined above: workflows = Matrix() diff --git a/Configuration/PyReleaseValidation/python/relval_ged.py b/Configuration/PyReleaseValidation/python/relval_ged.py index 95516a25ba8f4..02310290205fa 100644 --- a/Configuration/PyReleaseValidation/python/relval_ged.py +++ b/Configuration/PyReleaseValidation/python/relval_ged.py @@ -1,6 +1,7 @@ # import the definition of the steps and input files: -from Configuration.PyReleaseValidation.relval_steps import * +from Configuration.PyReleaseValidation.relval_steps import * +from .MatrixUtil import Matrix # here only define the workflows as a combination of the steps defined above: workflows = Matrix() diff --git a/Configuration/PyReleaseValidation/python/relval_generator.py b/Configuration/PyReleaseValidation/python/relval_generator.py index 57d35491e898d..2ba50844757cb 100644 --- a/Configuration/PyReleaseValidation/python/relval_generator.py +++ b/Configuration/PyReleaseValidation/python/relval_generator.py @@ -2,6 +2,7 @@ # # import the definition of the steps and input files: from Configuration.PyReleaseValidation.relval_steps import * +from .MatrixUtil import Matrix # here only define the workflows as a combination of the steps defined above: workflows = Matrix() diff --git a/Configuration/PyReleaseValidation/python/relval_gpu.py b/Configuration/PyReleaseValidation/python/relval_gpu.py index 60648515bf451..535837c9558c7 100644 --- a/Configuration/PyReleaseValidation/python/relval_gpu.py +++ b/Configuration/PyReleaseValidation/python/relval_gpu.py @@ -1,6 +1,7 @@ # import the definition of the steps and input files: from Configuration.PyReleaseValidation.relval_steps import * +from .MatrixUtil import Matrix # here only define the workflows as a combination of the steps defined above: workflows = Matrix() diff --git a/Configuration/PyReleaseValidation/python/relval_highstats.py b/Configuration/PyReleaseValidation/python/relval_highstats.py index 5808f63f0ebaa..4bdef2c1b940c 100644 --- a/Configuration/PyReleaseValidation/python/relval_highstats.py +++ b/Configuration/PyReleaseValidation/python/relval_highstats.py @@ -1,5 +1,6 @@ # import the definition of the steps and input files: from Configuration.PyReleaseValidation.relval_steps import * +from .MatrixUtil import Matrix # here only define the workflows as a combination of the steps defined above: workflows = Matrix() diff --git a/Configuration/PyReleaseValidation/python/relval_identity.py b/Configuration/PyReleaseValidation/python/relval_identity.py index 83a15d203e0a8..86d6d7f65ca40 100644 --- a/Configuration/PyReleaseValidation/python/relval_identity.py +++ b/Configuration/PyReleaseValidation/python/relval_identity.py @@ -1,6 +1,7 @@ # import the definition of the steps and input files: from Configuration.PyReleaseValidation.relval_steps import * +from .MatrixUtil import Matrix # here only define the workflows as a combination of the steps defined above: workflows = Matrix() diff --git a/Configuration/PyReleaseValidation/python/relval_machine.py b/Configuration/PyReleaseValidation/python/relval_machine.py index 9d19080779eab..02c409db88940 100644 --- a/Configuration/PyReleaseValidation/python/relval_machine.py +++ b/Configuration/PyReleaseValidation/python/relval_machine.py @@ -1,4 +1,4 @@ -from Configuration.PyReleaseValidation.relval_steps import Matrix, InputInfo, Steps +from .MatrixUtil import Matrix, InputInfo, Steps import os import json import collections diff --git a/Configuration/PyReleaseValidation/python/relval_nano.py b/Configuration/PyReleaseValidation/python/relval_nano.py index 4a77aa9f11379..042873a6f4f79 100644 --- a/Configuration/PyReleaseValidation/python/relval_nano.py +++ b/Configuration/PyReleaseValidation/python/relval_nano.py @@ -1,4 +1,5 @@ from Configuration.PyReleaseValidation.relval_steps import * +from .MatrixUtil import Matrix import math diff --git a/Configuration/PyReleaseValidation/python/relval_pileup.py b/Configuration/PyReleaseValidation/python/relval_pileup.py index 471521c3831f3..2b333d1cfae4d 100644 --- a/Configuration/PyReleaseValidation/python/relval_pileup.py +++ b/Configuration/PyReleaseValidation/python/relval_pileup.py @@ -1,5 +1,6 @@ # import the definition of the steps and input files: -from Configuration.PyReleaseValidation.relval_steps import * +from Configuration.PyReleaseValidation.relval_steps import * +from .MatrixUtil import Matrix # here only define the workflows as a combination of the steps defined above: workflows = Matrix() diff --git a/Configuration/PyReleaseValidation/python/relval_premix.py b/Configuration/PyReleaseValidation/python/relval_premix.py index 1d2704ba0b736..bc6806862a155 100644 --- a/Configuration/PyReleaseValidation/python/relval_premix.py +++ b/Configuration/PyReleaseValidation/python/relval_premix.py @@ -1,5 +1,6 @@ # import the definition of the steps and input files: -from Configuration.PyReleaseValidation.relval_steps import * +from Configuration.PyReleaseValidation.relval_steps import * +from .MatrixUtil import Matrix # here only define the workflows as a combination of the steps defined above: workflows = Matrix() diff --git a/Configuration/PyReleaseValidation/python/relval_production.py b/Configuration/PyReleaseValidation/python/relval_production.py index 6c511671c4e1c..2d04c86f90fbc 100644 --- a/Configuration/PyReleaseValidation/python/relval_production.py +++ b/Configuration/PyReleaseValidation/python/relval_production.py @@ -1,5 +1,6 @@ # import the definition of the steps and input files: -from Configuration.PyReleaseValidation.relval_steps import * +from Configuration.PyReleaseValidation.relval_steps import * +from .MatrixUtil import Matrix # here only define the workflows as a combination of the steps defined above: workflows = Matrix() diff --git a/Configuration/PyReleaseValidation/python/relval_standard.py b/Configuration/PyReleaseValidation/python/relval_standard.py index 94a2ce02c882c..eb157060e16fe 100644 --- a/Configuration/PyReleaseValidation/python/relval_standard.py +++ b/Configuration/PyReleaseValidation/python/relval_standard.py @@ -1,5 +1,6 @@ # import the definition of the steps and input files: from Configuration.PyReleaseValidation.relval_steps import * +from .MatrixUtil import Matrix # here only define the workflows as a combination of the steps defined above: workflows = Matrix() diff --git a/Configuration/PyReleaseValidation/python/relval_steps.py b/Configuration/PyReleaseValidation/python/relval_steps.py index bbda0d263fce8..c253030878c57 100644 --- a/Configuration/PyReleaseValidation/python/relval_steps.py +++ b/Configuration/PyReleaseValidation/python/relval_steps.py @@ -1,7 +1,8 @@ import sys +from copy import deepcopy from functools import partial -from .MatrixUtil import * +from .MatrixUtil import Steps, merge, remove, Kby, Mby, genvalid, InputInfo, selectedLS, stCond from Configuration.HLT.autoHLT import autoHLT from Configuration.AlCa.autoPCL import autoPCL @@ -3810,9 +3811,6 @@ def gen2024HiMix(fragment,howMuch): # '-s':'RECO,HLT:@fake,VALIDATION'}, # steps['RECO']]) -#add this line when testing from an input file that is not strictly GEN-SIM -#addForAll(step3,{'--hltProcess':'DIGI'}) - steps['ALCACOSD']={'--conditions':'auto:run1_data', '--datatier':'ALCARECO', '--eventcontent':'ALCARECO', @@ -4931,96 +4929,72 @@ def gen2024HiMix(fragment,howMuch): upgradeStepDict['GenHLBeamSpot'][k] = merge([{'--conditions' : gt+'_13TeV'}, upgradeStepDict['Gen'][k]]) upgradeStepDict['GenHLBeamSpot14'][k] = merge([{}, upgradeStepDict['Gen'][k]]) - upgradeStepDict['GenSim'][k]= {'-s' : 'GEN,SIM', - '-n' : 10, - '--conditions' : gt, - '--beamspot' : 'Realistic25ns13TeVEarly2017Collision', - '--datatier' : 'GEN-SIM', - '--eventcontent': 'FEVTDEBUG', - '--geometry' : geom - } + upgradeStepDict['GenSim'][k] = {'-s' : 'GEN,SIM', + '-n' : 10, + '--conditions' : gt, + '--beamspot' : 'Realistic25ns13TeVEarly2017Collision', + '--datatier' : 'GEN-SIM', + '--eventcontent': 'FEVTDEBUG', + '--geometry' : geom + } if beamspot is not None: upgradeStepDict['GenSim'][k]['--beamspot']=beamspot - upgradeStepDict['GenSimHLBeamSpot'][k]= {'-s' : 'GEN,SIM', - '-n' : 10, - '--conditions' : gt+'_13TeV', - '--beamspot' : 'DBrealisticHLLHC', - '--datatier' : 'GEN-SIM', - '--eventcontent': 'FEVTDEBUG', - '--geometry' : geom - } - - upgradeStepDict['GenSimHLBeamSpot14'][k]= {'-s' : 'GEN,SIM', - '-n' : 10, - '--conditions' : gt, - '--beamspot' : 'DBrealisticHLLHC', - '--datatier' : 'GEN-SIM', - '--eventcontent': 'FEVTDEBUG', - '--geometry' : geom - } - - upgradeStepDict['GenSimHLBeamSpotCloseBy'][k]= {'-s' : 'GEN,SIM', - '-n' : 10, - '--conditions' : gt, - '--beamspot' : 'CloseBy', - '--datatier' : 'GEN-SIM', - '--eventcontent': 'FEVTDEBUG', - '--geometry' : geom - } - - upgradeStepDict['Sim'][k]= {'-s' : 'SIM', - '-n' : 10, - '--conditions' : gt, - '--beamspot' : 'Realistic25ns13TeVEarly2017Collision', - '--datatier' : 'SIM', - '--eventcontent': 'FEVTDEBUG', - '--geometry' : geom - } + upgradeStepDict['GenSimCloseBy'][k] = deepcopy(upgradeStepDict['GenSim'][k]) + upgradeStepDict['GenSimCloseBy'][k]['--beamspot'] = 'CloseBy' + + upgradeStepDict['GenSimHLBeamSpot'][k] = {'-s' : 'GEN,SIM', + '-n' : 10, + '--conditions' : gt+'_13TeV', + '--beamspot' : 'DBrealisticHLLHC', + '--datatier' : 'GEN-SIM', + '--eventcontent': 'FEVTDEBUG', + '--geometry' : geom + } + + upgradeStepDict['GenSimHLBeamSpot14'][k] = deepcopy(upgradeStepDict['GenSimHLBeamSpot'][k]) + upgradeStepDict['GenSimHLBeamSpot14'][k]['--conditions'] = gt + + upgradeStepDict['GenSimHLBeamSpotCloseBy'][k] = upgradeStepDict['GenSimCloseBy'][k] + + upgradeStepDict['Sim'][k] = {'-s' : 'SIM', + '-n' : 10, + '--conditions' : gt, + '--beamspot' : 'Realistic25ns13TeVEarly2017Collision', + '--datatier' : 'SIM', + '--eventcontent': 'FEVTDEBUG', + '--geometry' : geom + } if beamspot is not None: upgradeStepDict['Sim'][k]['--beamspot']=beamspot upgradeStepDict['Digi'][k] = {'-s':'DIGI:pdigi_valid,L1,DIGI2RAW,HLT:%s'%(hltversion), - '--conditions':gt, - '--datatier':'GEN-SIM-DIGI-RAW', - '-n':'10', - '--eventcontent':'FEVTDEBUGHLT', - '--geometry' : geom - } + '--conditions':gt, + '--datatier':'GEN-SIM-DIGI-RAW', + '-n':'10', + '--eventcontent':'FEVTDEBUGHLT', + '--geometry' : geom + } - upgradeStepDict['DigiNoHLT'][k] = {'-s':'DIGI:pdigi_valid,L1,DIGI2RAW', - '--conditions':gt, - '--datatier':'GEN-SIM-DIGI-RAW', - '-n':'10', - '--eventcontent':'FEVTDEBUGHLT', - '--geometry' : geom - } + upgradeStepDict['DigiNoHLT'][k] = deepcopy(upgradeStepDict['Digi'][k]) + upgradeStepDict['DigiNoHLT'][k]['-s'] = 'DIGI:pdigi_valid,L1,DIGI2RAW' - upgradeStepDict['HLTOnly'][k] = {'-s':'HLT:%s'%(hltversion), - '--conditions':gt, - '--datatier':'GEN-SIM-DIGI-RAW', - '-n':'10', - '--eventcontent':'FEVTDEBUGHLT', - '--geometry' : geom, - } + upgradeStepDict['HLTOnly'][k] = deepcopy(upgradeStepDict['Digi'][k]) + upgradeStepDict['HLTOnly'][k]['-s'] = 'HLT:%s'%(hltversion) + # Adding Track trigger step in step2 - upgradeStepDict['DigiTrigger'][k] = {'-s':'DIGI:pdigi_valid,L1TrackTrigger,L1,L1P2GT,DIGI2RAW,HLT:%s'%(hltversion), - '--conditions':gt, - '--datatier':'GEN-SIM-DIGI-RAW', - '-n':'10', - '--eventcontent':'FEVTDEBUGHLT', - '--geometry' : geom - } + upgradeStepDict['DigiTrigger'][k] = deepcopy(upgradeStepDict['Digi'][k]) + upgradeStepDict['DigiTrigger'][k]['-s'] = 'DIGI:pdigi_valid,L1TrackTrigger,L1,L1P2GT,DIGI2RAW,HLT:%s'%(hltversion) upgradeStepDict['HLTRun3'][k] = {'-s':'HLT:%s'%(hltversion), - '--conditions':gt, - '--datatier':'GEN-SIM-DIGI-RAW', - '-n':'10', - '--eventcontent':'FEVTDEBUGHLT', - '--geometry' : geom, - '--outputCommands' : '"drop *_*_*_GEN,drop *_*_*_DIGI2RAW"' - } + '--conditions':gt, + '--datatier':'GEN-SIM-DIGI-RAW', + '-n':'10', + '--eventcontent':'FEVTDEBUGHLT', + '--geometry' : geom, + '--outputCommands' : '"drop *_*_*_GEN,drop *_*_*_DIGI2RAW"' + } upgradeStepDict['HLT75e33'][k] = {'-s':'HLT:@relvalRun4', '--processName':'HLTX', diff --git a/Configuration/PyReleaseValidation/python/relval_upgrade.py b/Configuration/PyReleaseValidation/python/relval_upgrade.py index 73b07d207dacc..53dea45e63aa0 100644 --- a/Configuration/PyReleaseValidation/python/relval_upgrade.py +++ b/Configuration/PyReleaseValidation/python/relval_upgrade.py @@ -1,5 +1,6 @@ # import the definition of the steps and input files: from Configuration.PyReleaseValidation.relval_steps import * +from .MatrixUtil import Matrix # here only define the workflows as a combination of the steps defined above: workflows = Matrix() @@ -40,8 +41,10 @@ def notForGenOnly(key,specialType): if 'HLBeamSpot' in step: if '14TeV' in frag: step = 'GenSimHLBeamSpot14' - if 'CloseBy' in frag or 'CE_E' in frag or 'CE_H' in frag: + elif 'CloseBy' in frag or 'CE_E' in frag or 'CE_H' in frag: step = 'GenSimHLBeamSpotCloseBy' + elif 'CloseBy' in frag or 'CE_E' in frag or 'CE_H' in frag: + step = 'GenSimCloseBy' stepMaker = makeStepNameSim elif 'Gen' in step: if 'HLBeamSpot' in step: diff --git a/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py b/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py index 857cdef447b26..228fe32aa02bc 100644 --- a/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py +++ b/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py @@ -1,4 +1,4 @@ -from copy import copy, deepcopy +from copy import deepcopy from collections import OrderedDict from .MatrixUtil import merge, Kby, Mby, check_dups import re @@ -207,6 +207,7 @@ def condition(self, fragment, stepList, key, hasHarvest): 'GenHLBeamSpot14', 'Sim', 'GenSim', + 'GenSimCloseBy', 'GenSimHLBeamSpot', 'GenSimHLBeamSpot14', 'GenSimHLBeamSpotCloseBy',