| 20 |
|
# init BlackWhiteListParser |
| 21 |
|
self.blackWhiteListParser = BlackWhiteListParser(cfg_params) |
| 22 |
|
|
| 23 |
< |
try: |
| 24 |
< |
self.MaxTarBallSize = float(self.cfg_params['EDG.maxtarballsize']) |
| 25 |
< |
except KeyError: |
| 26 |
< |
self.MaxTarBallSize = 9.5 |
| 23 |
> |
self.MaxTarBallSize = float(self.cfg_params.get('EDG.maxtarballsize',9.5)) |
| 24 |
|
|
| 25 |
|
# number of jobs requested to be created, limit obj splitting |
| 26 |
|
self.ncjobs = ncjobs |
| 61 |
|
|
| 62 |
|
### collect Data cards |
| 63 |
|
|
| 64 |
< |
try: |
| 68 |
< |
tmp = cfg_params['CMSSW.datasetpath'] |
| 69 |
< |
log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp) |
| 70 |
< |
if string.lower(tmp)=='none': |
| 71 |
< |
self.datasetPath = None |
| 72 |
< |
self.selectNoInput = 1 |
| 73 |
< |
else: |
| 74 |
< |
self.datasetPath = tmp |
| 75 |
< |
self.selectNoInput = 0 |
| 76 |
< |
except KeyError: |
| 64 |
> |
if not cfg_params.has_key('CMSSW.datasetpath'): |
| 65 |
|
msg = "Error: datasetpath not defined " |
| 66 |
|
raise CrabException(msg) |
| 67 |
+ |
tmp = cfg_params['CMSSW.datasetpath'] |
| 68 |
+ |
log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp) |
| 69 |
+ |
if string.lower(tmp)=='none': |
| 70 |
+ |
self.datasetPath = None |
| 71 |
+ |
self.selectNoInput = 1 |
| 72 |
+ |
else: |
| 73 |
+ |
self.datasetPath = tmp |
| 74 |
+ |
self.selectNoInput = 0 |
| 75 |
|
|
| 76 |
|
# ML monitoring |
| 77 |
|
# split dataset path style: /PreProdR3Minbias/SIM/GEN-SIM |
| 79 |
|
self.setParam_('dataset', 'None') |
| 80 |
|
self.setParam_('owner', 'None') |
| 81 |
|
else: |
| 82 |
+ |
## SL what is supposed to fail here? |
| 83 |
|
try: |
| 84 |
|
datasetpath_split = self.datasetPath.split("/") |
| 85 |
|
# standard style |
| 95 |
|
self.dataTiers = [] |
| 96 |
|
|
| 97 |
|
## now the application |
| 98 |
< |
try: |
| 99 |
< |
self.executable = cfg_params['CMSSW.executable'] |
| 100 |
< |
self.setParam_('exe', self.executable) |
| 104 |
< |
log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable) |
| 105 |
< |
msg = "Default executable cmsRun overridden. Switch to " + self.executable |
| 106 |
< |
log.debug(3,msg) |
| 107 |
< |
except KeyError: |
| 108 |
< |
self.executable = 'cmsRun' |
| 109 |
< |
self.setParam_('exe', self.executable) |
| 110 |
< |
msg = "User executable not defined. Use cmsRun" |
| 111 |
< |
log.debug(3,msg) |
| 112 |
< |
pass |
| 98 |
> |
self.executable = cfg_params.get('CMSSW.executable','cmsRun') |
| 99 |
> |
self.setParam_('exe', self.executable) |
| 100 |
> |
log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable) |
| 101 |
|
|
| 102 |
< |
try: |
| 115 |
< |
self.pset = cfg_params['CMSSW.pset'] |
| 116 |
< |
log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset) |
| 117 |
< |
if self.pset.lower() != 'none' : |
| 118 |
< |
if (not os.path.exists(self.pset)): |
| 119 |
< |
raise CrabException("User defined PSet file "+self.pset+" does not exist") |
| 120 |
< |
else: |
| 121 |
< |
self.pset = None |
| 122 |
< |
except KeyError: |
| 102 |
> |
if not cfg_params.has_key('CMSSW.pset'): |
| 103 |
|
raise CrabException("PSet file missing. Cannot run cmsRun ") |
| 104 |
+ |
self.pset = cfg_params['CMSSW.pset'] |
| 105 |
+ |
log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset) |
| 106 |
+ |
if self.pset.lower() != 'none' : |
| 107 |
+ |
if (not os.path.exists(self.pset)): |
| 108 |
+ |
raise CrabException("User defined PSet file "+self.pset+" does not exist") |
| 109 |
+ |
else: |
| 110 |
+ |
self.pset = None |
| 111 |
|
|
| 112 |
|
# output files |
| 113 |
|
## stuff which must be returned always via sandbox |
| 117 |
|
self.output_file_sandbox.append(self.fjrFileName) |
| 118 |
|
|
| 119 |
|
# other output files to be returned via sandbox or copied to SE |
| 120 |
< |
try: |
| 121 |
< |
self.output_file = [] |
| 122 |
< |
tmp = cfg_params['CMSSW.output_file'] |
| 123 |
< |
if tmp != '': |
| 124 |
< |
tmpOutFiles = string.split(cfg_params['CMSSW.output_file'],',') |
| 125 |
< |
log.debug(7, 'cmssw::cmssw(): output files '+str(tmpOutFiles)) |
| 126 |
< |
for tmp in tmpOutFiles: |
| 127 |
< |
tmp=string.strip(tmp) |
| 141 |
< |
self.output_file.append(tmp) |
| 142 |
< |
pass |
| 143 |
< |
else: |
| 144 |
< |
log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n") |
| 120 |
> |
self.output_file = [] |
| 121 |
> |
tmp = cfg_params.get('CMSSW.output_file',None) |
| 122 |
> |
if tmp : |
| 123 |
> |
tmpOutFiles = string.split(tmp,',') |
| 124 |
> |
log.debug(7, 'cmssw::cmssw(): output files '+str(tmpOutFiles)) |
| 125 |
> |
for tmp in tmpOutFiles: |
| 126 |
> |
tmp=string.strip(tmp) |
| 127 |
> |
self.output_file.append(tmp) |
| 128 |
|
pass |
| 129 |
< |
pass |
| 147 |
< |
except KeyError: |
| 129 |
> |
else: |
| 130 |
|
log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n") |
| 131 |
< |
pass |
| 131 |
> |
pass |
| 132 |
|
|
| 133 |
|
# script_exe file as additional file in inputSandbox |
| 134 |
< |
try: |
| 135 |
< |
self.scriptExe = cfg_params['USER.script_exe'] |
| 136 |
< |
if self.scriptExe != '': |
| 137 |
< |
if not os.path.isfile(self.scriptExe): |
| 138 |
< |
msg ="ERROR. file "+self.scriptExe+" not found" |
| 139 |
< |
raise CrabException(msg) |
| 158 |
< |
self.additional_inbox_files.append(string.strip(self.scriptExe)) |
| 159 |
< |
except KeyError: |
| 160 |
< |
self.scriptExe = '' |
| 134 |
> |
self.scriptExe = cfg_params.get('USER.script_exe',None) |
| 135 |
> |
if self.scriptExe : |
| 136 |
> |
if not os.path.isfile(self.scriptExe): |
| 137 |
> |
msg ="ERROR. file "+self.scriptExe+" not found" |
| 138 |
> |
raise CrabException(msg) |
| 139 |
> |
self.additional_inbox_files.append(string.strip(self.scriptExe)) |
| 140 |
|
|
| 141 |
|
#CarlosDaniele |
| 142 |
|
if self.datasetPath == None and self.pset == None and self.scriptExe == '' : |
| 144 |
|
raise CrabException(msg) |
| 145 |
|
|
| 146 |
|
## additional input files |
| 147 |
< |
try: |
| 147 |
> |
if cfg_params.has_key('USER.additional_input_files'): |
| 148 |
|
tmpAddFiles = string.split(cfg_params['USER.additional_input_files'],',') |
| 149 |
|
for tmp in tmpAddFiles: |
| 150 |
|
tmp = string.strip(tmp) |
| 168 |
|
pass |
| 169 |
|
pass |
| 170 |
|
common.logger.debug(5,"Additional input files: "+str(self.additional_inbox_files)) |
| 171 |
< |
except KeyError: |
| 193 |
< |
pass |
| 194 |
< |
|
| 195 |
< |
# files per job |
| 196 |
< |
try: |
| 197 |
< |
if (cfg_params['CMSSW.files_per_jobs']): |
| 198 |
< |
raise CrabException("files_per_jobs no longer supported. Quitting.") |
| 199 |
< |
except KeyError: |
| 200 |
< |
pass |
| 171 |
> |
pass |
| 172 |
|
|
| 173 |
|
## Events per job |
| 174 |
< |
try: |
| 174 |
> |
if cfg_params.has_key('CMSSW.events_per_job'): |
| 175 |
|
self.eventsPerJob =int( cfg_params['CMSSW.events_per_job']) |
| 176 |
|
self.selectEventsPerJob = 1 |
| 177 |
< |
except KeyError: |
| 177 |
> |
else: |
| 178 |
|
self.eventsPerJob = -1 |
| 179 |
|
self.selectEventsPerJob = 0 |
| 180 |
|
|
| 181 |
|
## number of jobs |
| 182 |
< |
try: |
| 182 |
> |
if cfg_params.has_key('CMSSW.number_of_jobs'): |
| 183 |
|
self.theNumberOfJobs =int( cfg_params['CMSSW.number_of_jobs']) |
| 184 |
|
self.selectNumberOfJobs = 1 |
| 185 |
< |
except KeyError: |
| 185 |
> |
else: |
| 186 |
|
self.theNumberOfJobs = 0 |
| 187 |
|
self.selectNumberOfJobs = 0 |
| 188 |
|
|
| 189 |
< |
try: |
| 189 |
> |
if cfg_params.has_key('CMSSW.total_number_of_events'): |
| 190 |
|
self.total_number_of_events = int(cfg_params['CMSSW.total_number_of_events']) |
| 191 |
|
self.selectTotalNumberEvents = 1 |
| 192 |
< |
except KeyError: |
| 192 |
> |
else: |
| 193 |
|
self.total_number_of_events = 0 |
| 194 |
|
self.selectTotalNumberEvents = 0 |
| 195 |
|
|
| 203 |
|
raise CrabException(msg) |
| 204 |
|
|
| 205 |
|
## source seed for pythia |
| 206 |
< |
try: |
| 236 |
< |
self.sourceSeed = int(cfg_params['CMSSW.pythia_seed']) |
| 237 |
< |
except KeyError: |
| 238 |
< |
self.sourceSeed = None |
| 239 |
< |
common.logger.debug(5,"No seed given") |
| 206 |
> |
self.sourceSeed = cfg_params.get('CMSSW.pythia_seed',None) |
| 207 |
|
|
| 208 |
< |
try: |
| 242 |
< |
self.sourceSeedVtx = int(cfg_params['CMSSW.vtx_seed']) |
| 243 |
< |
except KeyError: |
| 244 |
< |
self.sourceSeedVtx = None |
| 245 |
< |
common.logger.debug(5,"No vertex seed given") |
| 208 |
> |
self.sourceSeedVtx = cfg_params.get('CMSSW.vtx_seed',None) |
| 209 |
|
|
| 210 |
< |
try: |
| 248 |
< |
self.sourceSeedG4 = int(cfg_params['CMSSW.g4_seed']) |
| 249 |
< |
except KeyError: |
| 250 |
< |
self.sourceSeedG4 = None |
| 251 |
< |
common.logger.debug(5,"No g4 sim hits seed given") |
| 210 |
> |
self.sourceSeedG4 = cfg_params.get('CMSSW.g4_seed',None) |
| 211 |
|
|
| 212 |
< |
try: |
| 213 |
< |
self.sourceSeedMix = int(cfg_params['CMSSW.mix_seed']) |
| 214 |
< |
except KeyError: |
| 256 |
< |
self.sourceSeedMix = None |
| 257 |
< |
common.logger.debug(5,"No mix seed given") |
| 212 |
> |
self.sourceSeedMix = cfg_params.get('CMSSW.mix_seed',None) |
| 213 |
> |
|
| 214 |
> |
self.firstRun = cfg_params.get('CMSSW.first_run',None) |
| 215 |
|
|
| 259 |
– |
try: |
| 260 |
– |
self.firstRun = int(cfg_params['CMSSW.first_run']) |
| 261 |
– |
except KeyError: |
| 262 |
– |
self.firstRun = None |
| 263 |
– |
common.logger.debug(5,"No first run given") |
| 216 |
|
if self.pset != None: #CarlosDaniele |
| 217 |
|
import PsetManipulator as pp |
| 218 |
|
PsetEdit = pp.PsetManipulator(self.pset) #Daniele Pset |
| 219 |
|
|
| 220 |
|
# Copy/return |
| 221 |
|
|
| 222 |
< |
try: |
| 223 |
< |
self.copy_data = int(cfg_params['USER.copy_data']) |
| 272 |
< |
except KeyError: |
| 273 |
< |
self.copy_data = 0 |
| 274 |
< |
try: |
| 275 |
< |
self.return_data = int(cfg_params['USER.return_data']) |
| 276 |
< |
except KeyError: |
| 277 |
< |
self.return_data = 0 |
| 222 |
> |
self.copy_data = int(cfg_params.get('USER.copy_data',0)) |
| 223 |
> |
self.return_data = int(cfg_params.get('USER.return_data',0)) |
| 224 |
|
|
| 225 |
|
#DBSDLS-start |
| 226 |
|
## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code |
| 283 |
|
## Contact the DBS |
| 284 |
|
common.logger.message("Contacting Data Discovery Services ...") |
| 285 |
|
try: |
| 340 |
– |
|
| 286 |
|
self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params) |
| 287 |
|
self.pubdata.fetchDBSInfo() |
| 288 |
|
|
| 540 |
|
for range_jobs in noSiteBlock: |
| 541 |
|
msg += str(range_jobs) + virgola |
| 542 |
|
msg += '\n will not be submitted and this block of data can not be analyzed!\n' |
| 543 |
+ |
if self.cfg_params.has_key('EDG.se_white_list'): |
| 544 |
+ |
msg += 'WARNING: SE White List: '+self.cfg_params['EDG.se_white_list']+'\n' |
| 545 |
+ |
msg += '(Hint: By whitelisting you force the job to run at this particular site(s).\n' |
| 546 |
+ |
msg += 'Please check if the dataset is available at this site!)\n' |
| 547 |
+ |
if self.cfg_params.has_key('EDG.ce_white_list'): |
| 548 |
+ |
msg += 'WARNING: CE White List: '+self.cfg_params['EDG.ce_white_list']+'\n' |
| 549 |
+ |
msg += '(Hint: By whitelisting you force the job to run at this particular site(s).\n' |
| 550 |
+ |
msg += 'Please check if the dataset is available at this site!)\n' |
| 551 |
+ |
|
| 552 |
|
common.logger.message(msg) |
| 553 |
|
|
| 554 |
|
self.list_of_args = list_of_lists |
| 1115 |
|
txt += 'else\n' |
| 1116 |
|
txt += ' exit_status=60302\n' |
| 1117 |
|
txt += ' echo "ERROR: Output file '+fileWithSuffix+' not found"\n' |
| 1118 |
< |
if common.scheduler.boss_scheduler_name == 'condor_g': |
| 1118 |
> |
if common.scheduler.name() == 'CONDOR_G': |
| 1119 |
|
txt += ' if [ $middleware == OSG ]; then \n' |
| 1120 |
|
txt += ' echo "prepare dummy output file"\n' |
| 1121 |
|
txt += ' echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n' |
| 1138 |
|
txt += ' echo "ERROR: Output file '+fileWithSuffix+' not found"\n' |
| 1139 |
|
txt += ' echo "JOB_EXIT_STATUS = $exit_status"\n' |
| 1140 |
|
txt += ' output_exit_status=$exit_status\n' |
| 1141 |
< |
if common.scheduler.boss_scheduler_name == 'condor_g': |
| 1141 |
> |
if common.scheduler.name() == 'CONDOR_G': |
| 1142 |
|
txt += ' if [ $middleware == OSG ]; then \n' |
| 1143 |
|
txt += ' echo "prepare dummy output file"\n' |
| 1144 |
|
txt += ' echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n' |
| 1318 |
|
txt += 'echo "ProcessedDataset = $ProcessedDataset"\n' |
| 1319 |
|
txt += 'echo "FOR_LFN = $FOR_LFN" \n' |
| 1320 |
|
txt += 'echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n' |
| 1367 |
– |
#txt += 'echo "$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n' |
| 1368 |
– |
#txt += '$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n' |
| 1321 |
|
txt += 'echo "$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n' |
| 1322 |
|
txt += '$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n' |
| 1323 |
|
|
