| 2 |
|
from crab_logger import Logger |
| 3 |
|
from crab_exceptions import * |
| 4 |
|
from crab_util import * |
| 5 |
< |
from BlackWhiteListParser import BlackWhiteListParser |
| 5 |
> |
from BlackWhiteListParser import SEBlackWhiteListParser |
| 6 |
|
import common |
| 7 |
|
import Scram |
| 8 |
+ |
from LFNBaseName import * |
| 9 |
|
|
| 10 |
|
import os, string, glob |
| 11 |
|
|
| 12 |
|
class Cmssw(JobType): |
| 13 |
< |
def __init__(self, cfg_params, ncjobs): |
| 13 |
> |
def __init__(self, cfg_params, ncjobs,skip_blocks, isNew): |
| 14 |
|
JobType.__init__(self, 'CMSSW') |
| 15 |
|
common.logger.debug(3,'CMSSW::__init__') |
| 16 |
+ |
self.skip_blocks = skip_blocks |
| 17 |
|
|
| 18 |
|
self.argsList = [] |
| 19 |
|
|
| 20 |
|
self._params = {} |
| 21 |
|
self.cfg_params = cfg_params |
| 22 |
|
# init BlackWhiteListParser |
| 23 |
< |
self.blackWhiteListParser = BlackWhiteListParser(cfg_params) |
| 23 |
> |
self.blackWhiteListParser = SEBlackWhiteListParser(cfg_params) |
| 24 |
|
|
| 25 |
< |
self.MaxTarBallSize = float(self.cfg_params.get('EDG.maxtarballsize',9.5)) |
| 25 |
> |
### Temporary patch to automatically skip the ISB size check: |
| 26 |
> |
server=self.cfg_params.get('CRAB.server_name',None) |
| 27 |
> |
size = 9.5 |
| 28 |
> |
if server: size = 99999 |
| 29 |
> |
### D.S. |
| 30 |
> |
self.MaxTarBallSize = float(self.cfg_params.get('EDG.maxtarballsize',size)) |
| 31 |
|
|
| 32 |
|
# number of jobs requested to be created, limit obj splitting |
| 33 |
|
self.ncjobs = ncjobs |
| 40 |
|
self.executable = '' |
| 41 |
|
self.executable_arch = self.scram.getArch() |
| 42 |
|
self.tgz_name = 'default.tgz' |
| 36 |
– |
self.additional_tgz_name = 'additional.tgz' |
| 43 |
|
self.scriptName = 'CMSSW.sh' |
| 44 |
< |
self.pset = '' #scrip use case Da |
| 45 |
< |
self.datasetPath = '' #scrip use case Da |
| 44 |
> |
self.pset = '' |
| 45 |
> |
self.datasetPath = '' |
| 46 |
|
|
| 47 |
|
# set FJR file name |
| 48 |
|
self.fjrFileName = 'crab_fjr.xml' |
| 49 |
|
|
| 50 |
|
self.version = self.scram.getSWVersion() |
| 51 |
< |
|
| 52 |
< |
# |
| 53 |
< |
# Try to block creation in case of arch/version mismatch |
| 54 |
< |
# |
| 55 |
< |
|
| 56 |
< |
# a = string.split(self.version, "_") |
| 57 |
< |
# |
| 58 |
< |
# if int(a[1]) == 1 and (int(a[2]) < 5 and self.executable_arch.find('slc4') == 0): |
| 59 |
< |
# msg = "Warning: You are using %s version of CMSSW with %s architecture. \n--> Did you compile your libraries with SLC3? Otherwise you can find some problems running on SLC4 Grid nodes.\n"%(self.version, self.executable_arch) |
| 60 |
< |
# common.logger.message(msg) |
| 61 |
< |
# if int(a[1]) == 1 and (int(a[2]) >= 5 and self.executable_arch.find('slc3') == 0): |
| 56 |
< |
# msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch) |
| 57 |
< |
# raise CrabException(msg) |
| 58 |
< |
# |
| 59 |
< |
|
| 60 |
< |
self.setParam_('application', self.version) |
| 51 |
> |
version_array = self.version.split('_') |
| 52 |
> |
self.CMSSW_major = 0 |
| 53 |
> |
self.CMSSW_minor = 0 |
| 54 |
> |
self.CMSSW_patch = 0 |
| 55 |
> |
try: |
| 56 |
> |
self.CMSSW_major = int(version_array[1]) |
| 57 |
> |
self.CMSSW_minor = int(version_array[2]) |
| 58 |
> |
self.CMSSW_patch = int(version_array[3]) |
| 59 |
> |
except: |
| 60 |
> |
msg = "Cannot parse CMSSW version string: " + self.version + " for major and minor release number!" |
| 61 |
> |
raise CrabException(msg) |
| 62 |
|
|
| 63 |
|
### collect Data cards |
| 64 |
|
|
| 65 |
|
if not cfg_params.has_key('CMSSW.datasetpath'): |
| 66 |
|
msg = "Error: datasetpath not defined " |
| 67 |
|
raise CrabException(msg) |
| 68 |
+ |
|
| 69 |
+ |
### Temporary: added to remove input file control in the case of PU |
| 70 |
+ |
self.dataset_pu = cfg_params.get('CMSSW.dataset_pu', None) |
| 71 |
+ |
|
| 72 |
|
tmp = cfg_params['CMSSW.datasetpath'] |
| 73 |
|
log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp) |
| 74 |
|
if string.lower(tmp)=='none': |
| 78 |
|
self.datasetPath = tmp |
| 79 |
|
self.selectNoInput = 0 |
| 80 |
|
|
| 76 |
– |
# ML monitoring |
| 77 |
– |
# split dataset path style: /PreProdR3Minbias/SIM/GEN-SIM |
| 78 |
– |
if not self.datasetPath: |
| 79 |
– |
self.setParam_('dataset', 'None') |
| 80 |
– |
self.setParam_('owner', 'None') |
| 81 |
– |
else: |
| 82 |
– |
## SL what is supposed to fail here? |
| 83 |
– |
try: |
| 84 |
– |
datasetpath_split = self.datasetPath.split("/") |
| 85 |
– |
# standard style |
| 86 |
– |
self.setParam_('datasetFull', self.datasetPath) |
| 87 |
– |
self.setParam_('dataset', datasetpath_split[1]) |
| 88 |
– |
self.setParam_('owner', datasetpath_split[2]) |
| 89 |
– |
except: |
| 90 |
– |
self.setParam_('dataset', self.datasetPath) |
| 91 |
– |
self.setParam_('owner', self.datasetPath) |
| 92 |
– |
|
| 93 |
– |
self.setParam_('taskId', common._db.queryTask('name')) ## new BL--DS |
| 94 |
– |
|
| 81 |
|
self.dataTiers = [] |
| 82 |
< |
|
| 82 |
> |
self.debugWrap = '' |
| 83 |
> |
self.debug_wrapper = cfg_params.get('USER.debug_wrapper',False) |
| 84 |
> |
if self.debug_wrapper: self.debugWrap='--debug' |
| 85 |
|
## now the application |
| 86 |
|
self.executable = cfg_params.get('CMSSW.executable','cmsRun') |
| 99 |
– |
self.setParam_('exe', self.executable) |
| 87 |
|
log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable) |
| 88 |
|
|
| 89 |
|
if not cfg_params.has_key('CMSSW.pset'): |
| 104 |
|
self.output_file_sandbox.append(self.fjrFileName) |
| 105 |
|
|
| 106 |
|
# other output files to be returned via sandbox or copied to SE |
| 107 |
+ |
outfileflag = False |
| 108 |
|
self.output_file = [] |
| 109 |
|
tmp = cfg_params.get('CMSSW.output_file',None) |
| 110 |
|
if tmp : |
| 111 |
< |
tmpOutFiles = string.split(tmp,',') |
| 112 |
< |
log.debug(7, 'cmssw::cmssw(): output files '+str(tmpOutFiles)) |
| 113 |
< |
for tmp in tmpOutFiles: |
| 114 |
< |
tmp=string.strip(tmp) |
| 127 |
< |
self.output_file.append(tmp) |
| 128 |
< |
pass |
| 129 |
< |
else: |
| 130 |
< |
log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n") |
| 131 |
< |
pass |
| 111 |
> |
self.output_file = [x.strip() for x in tmp.split(',')] |
| 112 |
> |
outfileflag = True #output found |
| 113 |
> |
#else: |
| 114 |
> |
# log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n") |
| 115 |
|
|
| 116 |
|
# script_exe file as additional file in inputSandbox |
| 117 |
|
self.scriptExe = cfg_params.get('USER.script_exe',None) |
| 118 |
|
if self.scriptExe : |
| 119 |
< |
if not os.path.isfile(self.scriptExe): |
| 120 |
< |
msg ="ERROR. file "+self.scriptExe+" not found" |
| 121 |
< |
raise CrabException(msg) |
| 122 |
< |
self.additional_inbox_files.append(string.strip(self.scriptExe)) |
| 119 |
> |
if not os.path.isfile(self.scriptExe): |
| 120 |
> |
msg ="ERROR. file "+self.scriptExe+" not found" |
| 121 |
> |
raise CrabException(msg) |
| 122 |
> |
self.additional_inbox_files.append(string.strip(self.scriptExe)) |
| 123 |
|
|
| 141 |
– |
#CarlosDaniele |
| 124 |
|
if self.datasetPath == None and self.pset == None and self.scriptExe == '' : |
| 125 |
< |
msg ="Error. script_exe not defined" |
| 126 |
< |
raise CrabException(msg) |
| 125 |
> |
msg ="Error. script_exe not defined" |
| 126 |
> |
raise CrabException(msg) |
| 127 |
> |
|
| 128 |
> |
# use parent files... |
| 129 |
> |
self.useParent = self.cfg_params.get('CMSSW.use_parent',False) |
| 130 |
|
|
| 131 |
|
## additional input files |
| 132 |
|
if cfg_params.has_key('USER.additional_input_files'): |
| 146 |
|
if not os.path.exists(file): |
| 147 |
|
raise CrabException("Additional input file not found: "+file) |
| 148 |
|
pass |
| 164 |
– |
# fname = string.split(file, '/')[-1] |
| 165 |
– |
# storedFile = common.work_space.pathForTgz()+'share/'+fname |
| 166 |
– |
# shutil.copyfile(file, storedFile) |
| 149 |
|
self.additional_inbox_files.append(string.strip(file)) |
| 150 |
|
pass |
| 151 |
|
pass |
| 171 |
|
if cfg_params.has_key('CMSSW.total_number_of_events'): |
| 172 |
|
self.total_number_of_events = int(cfg_params['CMSSW.total_number_of_events']) |
| 173 |
|
self.selectTotalNumberEvents = 1 |
| 174 |
+ |
if self.selectNumberOfJobs == 1: |
| 175 |
+ |
if (self.total_number_of_events != -1) and int(self.total_number_of_events) < int(self.theNumberOfJobs): |
| 176 |
+ |
msg = 'Must specify at least one event per job. total_number_of_events > number_of_jobs ' |
| 177 |
+ |
raise CrabException(msg) |
| 178 |
|
else: |
| 179 |
|
self.total_number_of_events = 0 |
| 180 |
|
self.selectTotalNumberEvents = 0 |
| 181 |
|
|
| 182 |
< |
if self.pset != None: #CarlosDaniele |
| 182 |
> |
if self.pset != None: |
| 183 |
|
if ( (self.selectTotalNumberEvents + self.selectEventsPerJob + self.selectNumberOfJobs) != 2 ): |
| 184 |
|
msg = 'Must define exactly two of total_number_of_events, events_per_job, or number_of_jobs.' |
| 185 |
|
raise CrabException(msg) |
| 202 |
|
tmp.strip() |
| 203 |
|
self.incrementSeeds.append(tmp) |
| 204 |
|
|
| 205 |
< |
## Old method of dealing with seeds |
| 206 |
< |
## FUTURE: This is for old CMSSW and old CRAB. Can throw exceptions after a couple of CRAB releases and then |
| 221 |
< |
## remove |
| 222 |
< |
self.sourceSeed = cfg_params.get('CMSSW.pythia_seed',None) |
| 223 |
< |
if self.sourceSeed: |
| 224 |
< |
print "pythia_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds." |
| 225 |
< |
self.incrementSeeds.append('sourceSeed') |
| 226 |
< |
|
| 205 |
> |
## FUTURE: Can remove in CRAB 2.4.0 |
| 206 |
> |
self.sourceSeed = cfg_params.get('CMSSW.pythia_seed',None) |
| 207 |
|
self.sourceSeedVtx = cfg_params.get('CMSSW.vtx_seed',None) |
| 208 |
< |
if self.sourceSeedVtx: |
| 229 |
< |
print "vtx_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds." |
| 230 |
< |
self.incrementSeeds.append('VtxSmeared') |
| 231 |
< |
|
| 232 |
< |
self.sourceSeedG4 = cfg_params.get('CMSSW.g4_seed',None) |
| 233 |
< |
if self.sourceSeedG4: |
| 234 |
< |
print "g4_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds." |
| 235 |
< |
self.incrementSeeds.append('g4SimHits') |
| 236 |
< |
|
| 208 |
> |
self.sourceSeedG4 = cfg_params.get('CMSSW.g4_seed',None) |
| 209 |
|
self.sourceSeedMix = cfg_params.get('CMSSW.mix_seed',None) |
| 210 |
< |
if self.sourceSeedMix: |
| 211 |
< |
print "mix_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds." |
| 212 |
< |
self.incrementSeeds.append('mix') |
| 210 |
> |
if self.sourceSeed or self.sourceSeedVtx or self.sourceSeedG4 or self.sourceSeedMix: |
| 211 |
> |
msg = 'pythia_seed, vtx_seed, g4_seed, and mix_seed are no longer valid settings. You must use increment_seeds or preserve_seeds' |
| 212 |
> |
raise CrabException(msg) |
| 213 |
|
|
| 214 |
|
self.firstRun = cfg_params.get('CMSSW.first_run',None) |
| 215 |
|
|
| 244 |
– |
if self.pset != None: #CarlosDaniele |
| 245 |
– |
import PsetManipulator as pp |
| 246 |
– |
PsetEdit = pp.PsetManipulator(self.pset) #Daniele Pset |
| 247 |
– |
|
| 216 |
|
# Copy/return |
| 249 |
– |
|
| 217 |
|
self.copy_data = int(cfg_params.get('USER.copy_data',0)) |
| 218 |
|
self.return_data = int(cfg_params.get('USER.return_data',0)) |
| 219 |
|
|
| 229 |
|
blockSites = self.DataDiscoveryAndLocation(cfg_params) |
| 230 |
|
#DBSDLS-end |
| 231 |
|
|
| 265 |
– |
self.tgzNameWithPath = self.getTarBall(self.executable) |
| 266 |
– |
|
| 232 |
|
## Select Splitting |
| 233 |
|
if self.selectNoInput: |
| 234 |
< |
if self.pset == None: #CarlosDaniele |
| 234 |
> |
if self.pset == None: |
| 235 |
|
self.jobSplittingForScript() |
| 236 |
|
else: |
| 237 |
|
self.jobSplittingNoInput() |
| 238 |
|
else: |
| 239 |
|
self.jobSplittingByBlocks(blockSites) |
| 240 |
|
|
| 241 |
< |
# modify Pset |
| 242 |
< |
if self.pset != None: #CarlosDaniele |
| 243 |
< |
try: |
| 244 |
< |
# Add FrameworkJobReport to parameter-set, set max events. |
| 245 |
< |
# Reset later for data jobs by writeCFG which does all modifications |
| 246 |
< |
PsetEdit.addCrabFJR(self.fjrFileName) |
| 247 |
< |
PsetEdit.maxEvent(self.eventsPerJob) |
| 248 |
< |
PsetEdit.psetWriter(self.configFilename()) |
| 249 |
< |
except: |
| 250 |
< |
msg='Error while manipuliating ParameterSet: exiting...' |
| 251 |
< |
raise CrabException(msg) |
| 241 |
> |
# modify Pset only the first time |
| 242 |
> |
if isNew: |
| 243 |
> |
if self.pset != None: |
| 244 |
> |
import PsetManipulator as pp |
| 245 |
> |
PsetEdit = pp.PsetManipulator(self.pset) |
| 246 |
> |
try: |
| 247 |
> |
# Add FrameworkJobReport to parameter-set, set max events. |
| 248 |
> |
# Reset later for data jobs by writeCFG which does all modifications |
| 249 |
> |
PsetEdit.addCrabFJR(self.fjrFileName) # FUTURE: Job report addition not needed by CMSSW>1.5 |
| 250 |
> |
PsetEdit.maxEvent(self.eventsPerJob) |
| 251 |
> |
PsetEdit.psetWriter(self.configFilename()) |
| 252 |
> |
## If present, add TFileService to output files |
| 253 |
> |
if not int(cfg_params.get('CMSSW.skip_TFileService_output',0)): |
| 254 |
> |
tfsOutput = PsetEdit.getTFileService() |
| 255 |
> |
if tfsOutput: |
| 256 |
> |
if tfsOutput in self.output_file: |
| 257 |
> |
common.logger.debug(5,"Output from TFileService "+tfsOutput+" already in output files") |
| 258 |
> |
else: |
| 259 |
> |
outfileflag = True #output found |
| 260 |
> |
self.output_file.append(tfsOutput) |
| 261 |
> |
common.logger.message("Adding "+tfsOutput+" to output files (from TFileService)") |
| 262 |
> |
pass |
| 263 |
> |
pass |
| 264 |
> |
## If present and requested, add PoolOutputModule to output files |
| 265 |
> |
if int(cfg_params.get('CMSSW.get_edm_output',0)): |
| 266 |
> |
edmOutput = PsetEdit.getPoolOutputModule() |
| 267 |
> |
if edmOutput: |
| 268 |
> |
if edmOutput in self.output_file: |
| 269 |
> |
common.logger.debug(5,"Output from PoolOutputModule "+edmOutput+" already in output files") |
| 270 |
> |
else: |
| 271 |
> |
self.output_file.append(edmOutput) |
| 272 |
> |
common.logger.message("Adding "+edmOutput+" to output files (from PoolOutputModule)") |
| 273 |
> |
pass |
| 274 |
> |
pass |
| 275 |
> |
except CrabException: |
| 276 |
> |
msg='Error while manipulating ParameterSet: exiting...' |
| 277 |
> |
raise CrabException(msg) |
| 278 |
> |
## Prepare inputSandbox TarBall (only the first time) |
| 279 |
> |
self.tgzNameWithPath = self.getTarBall(self.executable) |
| 280 |
|
|
| 281 |
|
def DataDiscoveryAndLocation(self, cfg_params): |
| 282 |
|
|
| 289 |
|
## Contact the DBS |
| 290 |
|
common.logger.message("Contacting Data Discovery Services ...") |
| 291 |
|
try: |
| 292 |
< |
self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params) |
| 292 |
> |
self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params,self.skip_blocks) |
| 293 |
|
self.pubdata.fetchDBSInfo() |
| 294 |
|
|
| 295 |
|
except DataDiscovery.NotExistingDatasetError, ex : |
| 305 |
|
self.filesbyblock=self.pubdata.getFiles() |
| 306 |
|
self.eventsbyblock=self.pubdata.getEventsPerBlock() |
| 307 |
|
self.eventsbyfile=self.pubdata.getEventsPerFile() |
| 308 |
+ |
self.parentFiles=self.pubdata.getParent() |
| 309 |
|
|
| 310 |
|
## get max number of events |
| 311 |
< |
self.maxEvents=self.pubdata.getMaxEvents() ## self.maxEvents used in Creator.py |
| 311 |
> |
self.maxEvents=self.pubdata.getMaxEvents() |
| 312 |
|
|
| 313 |
|
## Contact the DLS and build a list of sites hosting the fileblocks |
| 314 |
|
try: |
| 332 |
|
|
| 333 |
|
return sites |
| 334 |
|
|
| 341 |
– |
# to Be Removed DS -- BL |
| 342 |
– |
# def setArgsList(self, argsList): |
| 343 |
– |
# self.argsList = argsList |
| 344 |
– |
|
| 335 |
|
def jobSplittingByBlocks(self, blockSites): |
| 336 |
|
""" |
| 337 |
|
Perform job splitting. Jobs run over an integer number of files |
| 382 |
|
else : |
| 383 |
|
totalNumberOfJobs = self.ncjobs |
| 384 |
|
|
| 395 |
– |
|
| 385 |
|
blocks = blockSites.keys() |
| 386 |
|
blockCount = 0 |
| 387 |
|
# Backup variable in case self.maxEvents counted events in a non-included block |
| 422 |
|
|
| 423 |
|
# ---- Iterate over the files in the block until we've met the requested ---- # |
| 424 |
|
# ---- total # of events or we've gone over all the files in this block ---- # |
| 425 |
+ |
pString='' |
| 426 |
|
while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) and (jobCount < totalNumberOfJobs) ): |
| 427 |
|
file = files[fileCount] |
| 428 |
+ |
if self.useParent: |
| 429 |
+ |
parent = self.parentFiles[file] |
| 430 |
+ |
for f in parent : |
| 431 |
+ |
pString += '\\\"' + f + '\\\"\,' |
| 432 |
+ |
common.logger.debug(6, "File "+str(file)+" has the following parents: "+str(parent)) |
| 433 |
+ |
common.logger.write("File "+str(file)+" has the following parents: "+str(parent)) |
| 434 |
|
if newFile : |
| 435 |
|
try: |
| 436 |
|
numEventsInFile = self.eventsbyfile[file] |
| 443 |
|
except KeyError: |
| 444 |
|
common.logger.message("File "+str(file)+" has unknown number of events: skipping") |
| 445 |
|
|
| 446 |
< |
|
| 446 |
> |
eventsPerJobRequested = min(eventsPerJobRequested, eventsRemaining) |
| 447 |
|
# if less events in file remain than eventsPerJobRequested |
| 448 |
< |
if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested ) : |
| 448 |
> |
if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested): |
| 449 |
|
# if last file in block |
| 450 |
|
if ( fileCount == numFilesInBlock-1 ) : |
| 451 |
|
# end job using last file, use remaining events in block |
| 452 |
|
# close job and touch new file |
| 453 |
|
fullString = parString[:-2] |
| 454 |
< |
list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)]) |
| 454 |
> |
if self.useParent: |
| 455 |
> |
fullParentString = pString[:-2] |
| 456 |
> |
list_of_lists.append([fullString,fullParentString,str(-1),str(jobSkipEventCount)]) |
| 457 |
> |
else: |
| 458 |
> |
list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)]) |
| 459 |
|
common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(filesEventCount - jobSkipEventCount)+" events (last file in block).") |
| 460 |
|
self.jobDestination.append(blockSites[block]) |
| 461 |
|
common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount])) |
| 467 |
|
eventsRemaining = eventsRemaining - filesEventCount + jobSkipEventCount |
| 468 |
|
jobSkipEventCount = 0 |
| 469 |
|
# reset file |
| 470 |
+ |
pString = "" |
| 471 |
|
parString = "" |
| 472 |
|
filesEventCount = 0 |
| 473 |
|
newFile = 1 |
| 480 |
|
elif ( filesEventCount - jobSkipEventCount == eventsPerJobRequested ) : |
| 481 |
|
# close job and touch new file |
| 482 |
|
fullString = parString[:-2] |
| 483 |
< |
list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)]) |
| 483 |
> |
if self.useParent: |
| 484 |
> |
fullParentString = pString[:-2] |
| 485 |
> |
list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount)]) |
| 486 |
> |
else: |
| 487 |
> |
list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)]) |
| 488 |
|
common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.") |
| 489 |
|
self.jobDestination.append(blockSites[block]) |
| 490 |
|
common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount])) |
| 495 |
|
eventsRemaining = eventsRemaining - eventsPerJobRequested |
| 496 |
|
jobSkipEventCount = 0 |
| 497 |
|
# reset file |
| 498 |
+ |
pString = "" |
| 499 |
|
parString = "" |
| 500 |
|
filesEventCount = 0 |
| 501 |
|
newFile = 1 |
| 505 |
|
else : |
| 506 |
|
# close job but don't touch new file |
| 507 |
|
fullString = parString[:-2] |
| 508 |
< |
list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)]) |
| 508 |
> |
if self.useParent: |
| 509 |
> |
fullParentString = pString[:-2] |
| 510 |
> |
list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount)]) |
| 511 |
> |
else: |
| 512 |
> |
list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)]) |
| 513 |
|
common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.") |
| 514 |
|
self.jobDestination.append(blockSites[block]) |
| 515 |
|
common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount])) |
| 523 |
|
jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file]) |
| 524 |
|
# remove all but the last file |
| 525 |
|
filesEventCount = self.eventsbyfile[file] |
| 526 |
+ |
if self.useParent: |
| 527 |
+ |
for f in parent : pString += '\\\"' + f + '\\\"\,' |
| 528 |
|
parString = '\\\"' + file + '\\\"\,' |
| 529 |
|
pass # END if |
| 530 |
|
pass # END while (iterate over files in the block) |
| 545 |
|
for block in blocks: |
| 546 |
|
if block in jobsOfBlock.keys() : |
| 547 |
|
blockCounter += 1 |
| 548 |
< |
screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),','.join(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block))) |
| 548 |
> |
screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]), |
| 549 |
> |
','.join(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block))) |
| 550 |
|
if len(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)) == 0: |
| 551 |
|
noSiteBlock.append( spanRanges(jobsOfBlock[block]) ) |
| 552 |
|
bloskNoSite.append( blockCounter ) |
| 633 |
|
return |
| 634 |
|
|
| 635 |
|
|
| 636 |
< |
def jobSplittingForScript(self):#CarlosDaniele |
| 636 |
> |
def jobSplittingForScript(self): |
| 637 |
|
""" |
| 638 |
|
Perform job splitting based on number of job |
| 639 |
|
""" |
| 649 |
|
# argument is seed number.$i |
| 650 |
|
self.list_of_args = [] |
| 651 |
|
for i in range(self.total_number_of_jobs): |
| 639 |
– |
## Since there is no input, any site is good |
| 640 |
– |
# self.jobDestination.append(["Any"]) |
| 652 |
|
self.jobDestination.append([""]) |
| 642 |
– |
## no random seed |
| 653 |
|
self.list_of_args.append([str(i)]) |
| 654 |
|
return |
| 655 |
|
|
| 656 |
< |
def split(self, jobParams): |
| 656 |
> |
def split(self, jobParams,firstJobID): |
| 657 |
|
|
| 648 |
– |
#### Fabio |
| 658 |
|
njobs = self.total_number_of_jobs |
| 659 |
|
arglist = self.list_of_args |
| 660 |
|
# create the empty structure |
| 663 |
|
|
| 664 |
|
listID=[] |
| 665 |
|
listField=[] |
| 666 |
< |
for job in range(njobs): |
| 667 |
< |
jobParams[job] = arglist[job] |
| 666 |
> |
for id in range(njobs): |
| 667 |
> |
job = id + int(firstJobID) |
| 668 |
> |
jobParams[id] = arglist[id] |
| 669 |
|
listID.append(job+1) |
| 670 |
|
job_ToSave ={} |
| 671 |
|
concString = ' ' |
| 672 |
|
argu='' |
| 673 |
< |
if len(jobParams[job]): |
| 674 |
< |
argu += concString.join(jobParams[job] ) |
| 675 |
< |
job_ToSave['arguments']= str(job+1)+' '+argu## new BL--DS |
| 676 |
< |
job_ToSave['dlsDestination']= self.jobDestination[job]## new BL--DS |
| 667 |
< |
#common._db.updateJob_(job,job_ToSave)## new BL--DS |
| 673 |
> |
if len(jobParams[id]): |
| 674 |
> |
argu += concString.join(jobParams[id] ) |
| 675 |
> |
job_ToSave['arguments']= str(job+1)+' '+argu |
| 676 |
> |
job_ToSave['dlsDestination']= self.jobDestination[id] |
| 677 |
|
listField.append(job_ToSave) |
| 678 |
|
msg="Job "+str(job)+" Arguments: "+str(job+1)+" "+argu+"\n" \ |
| 679 |
< |
+" Destination: "+str(self.jobDestination[job]) |
| 679 |
> |
+" Destination: "+str(self.jobDestination[id]) |
| 680 |
|
common.logger.debug(5,msg) |
| 681 |
< |
#common.logger.debug(5,"Job "+str(job)+" Destination: "+str(self.jobDestination[job])) |
| 682 |
< |
common._db.updateJob_(listID,listField)## new BL--DS |
| 674 |
< |
## Pay Attention Here....DS--BL |
| 675 |
< |
self.argsList = (len(jobParams[1])+1) |
| 681 |
> |
common._db.updateJob_(listID,listField) |
| 682 |
> |
self.argsList = (len(jobParams[0])+1) |
| 683 |
|
|
| 684 |
|
return |
| 678 |
– |
# |
| 679 |
– |
# def getJobTypeArguments(self, nj, sched): |
| 680 |
– |
# result = '' |
| 681 |
– |
# jobs=[] |
| 682 |
– |
# jobs.append(nj) |
| 683 |
– |
# for i in common._db.queryJob('arguments',jobs):## BL--DS |
| 684 |
– |
# result=result+str(i)+" " |
| 685 |
– |
# return result |
| 685 |
|
|
| 686 |
|
def numberOfJobs(self): |
| 688 |
– |
# Fabio |
| 687 |
|
return self.total_number_of_jobs |
| 688 |
|
|
| 689 |
|
def getTarBall(self, exe): |
| 690 |
|
""" |
| 691 |
|
Return the TarBall with lib and exe |
| 692 |
|
""" |
| 695 |
– |
|
| 696 |
– |
# if it exist, just return it |
| 697 |
– |
# |
| 698 |
– |
# Marco. Let's start to use relative path for Boss XML files |
| 699 |
– |
# |
| 693 |
|
self.tgzNameWithPath = common.work_space.pathForTgz()+'share/'+self.tgz_name |
| 694 |
|
if os.path.exists(self.tgzNameWithPath): |
| 695 |
|
return self.tgzNameWithPath |
| 703 |
|
|
| 704 |
|
# First of all declare the user Scram area |
| 705 |
|
swArea = self.scram.getSWArea_() |
| 713 |
– |
#print "swArea = ", swArea |
| 714 |
– |
# swVersion = self.scram.getSWVersion() |
| 715 |
– |
# print "swVersion = ", swVersion |
| 706 |
|
swReleaseTop = self.scram.getReleaseTop_() |
| 717 |
– |
#print "swReleaseTop = ", swReleaseTop |
| 707 |
|
|
| 708 |
|
## check if working area is release top |
| 709 |
|
if swReleaseTop == '' or swArea == swReleaseTop: |
| 710 |
+ |
common.logger.debug(3,"swArea = "+swArea+" swReleaseTop ="+swReleaseTop) |
| 711 |
|
return |
| 712 |
|
|
| 713 |
|
import tarfile |
| 749 |
|
tar.add(module,moduleDir) |
| 750 |
|
|
| 751 |
|
## Now check if any data dir(s) is present |
| 752 |
< |
swAreaLen=len(swArea) |
| 753 |
< |
for root, dirs, files in os.walk(swArea): |
| 754 |
< |
if "data" in dirs: |
| 755 |
< |
common.logger.debug(5,"data "+root+"/data"+" to be tarred") |
| 756 |
< |
tar.add(root+"/data",root[swAreaLen:]+"/data") |
| 752 |
> |
self.dataExist = False |
| 753 |
> |
todo_list = [(i, i) for i in os.listdir(swArea+"/src")] |
| 754 |
> |
while len(todo_list): |
| 755 |
> |
entry, name = todo_list.pop() |
| 756 |
> |
if name.startswith('crab_0_') or name.startswith('.') or name == 'CVS': |
| 757 |
> |
continue |
| 758 |
> |
if os.path.isdir(swArea+"/src/"+entry): |
| 759 |
> |
entryPath = entry + '/' |
| 760 |
> |
todo_list += [(entryPath + i, i) for i in os.listdir(swArea+"/src/"+entry)] |
| 761 |
> |
if name == 'data': |
| 762 |
> |
self.dataExist=True |
| 763 |
> |
common.logger.debug(5,"data "+entry+" to be tarred") |
| 764 |
> |
tar.add(swArea+"/src/"+entry,"src/"+entry) |
| 765 |
> |
pass |
| 766 |
> |
pass |
| 767 |
> |
|
| 768 |
> |
### CMSSW ParameterSet |
| 769 |
> |
if not self.pset is None: |
| 770 |
> |
cfg_file = common.work_space.jobDir()+self.configFilename() |
| 771 |
> |
tar.add(cfg_file,self.configFilename()) |
| 772 |
> |
common.logger.debug(5,"File added to "+self.tgzNameWithPath+" : "+str(tar.getnames())) |
| 773 |
|
|
| 774 |
|
|
| 775 |
|
## Add ProdCommon dir to tar |
| 776 |
< |
prodcommonDir = 'ProdCommon' |
| 777 |
< |
prodcommonPath = os.environ['CRABDIR'] + '/' + 'ProdCommon' |
| 778 |
< |
if os.path.isdir(prodcommonPath): |
| 779 |
< |
tar.add(prodcommonPath,prodcommonDir) |
| 776 |
> |
prodcommonDir = './' |
| 777 |
> |
prodcommonPath = os.environ['CRABDIR'] + '/' + 'external/' |
| 778 |
> |
neededStuff = ['ProdCommon/__init__.py','ProdCommon/FwkJobRep', 'ProdCommon/CMSConfigTools','ProdCommon/Core','ProdCommon/MCPayloads', 'IMProv'] |
| 779 |
> |
for file in neededStuff: |
| 780 |
> |
tar.add(prodcommonPath+file,prodcommonDir+file) |
| 781 |
> |
common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames())) |
| 782 |
|
|
| 783 |
+ |
##### ML stuff |
| 784 |
+ |
ML_file_list=['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py'] |
| 785 |
+ |
path=os.environ['CRABDIR'] + '/python/' |
| 786 |
+ |
for file in ML_file_list: |
| 787 |
+ |
tar.add(path+file,file) |
| 788 |
+ |
common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames())) |
| 789 |
+ |
|
| 790 |
+ |
##### Utils |
| 791 |
+ |
Utils_file_list=['parseCrabFjr.py','writeCfg.py', 'fillCrabFjr.py'] |
| 792 |
+ |
for file in Utils_file_list: |
| 793 |
+ |
tar.add(path+file,file) |
| 794 |
|
common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames())) |
| 795 |
+ |
|
| 796 |
+ |
##### AdditionalFiles |
| 797 |
+ |
for file in self.additional_inbox_files: |
| 798 |
+ |
tar.add(file,string.split(file,'/')[-1]) |
| 799 |
+ |
common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames())) |
| 800 |
+ |
|
| 801 |
|
tar.close() |
| 802 |
< |
except : |
| 803 |
< |
raise CrabException('Could not create tar-ball') |
| 802 |
> |
except IOError: |
| 803 |
> |
raise CrabException('Could not create tar-ball '+self.tgzNameWithPath) |
| 804 |
> |
except tarfile.TarError: |
| 805 |
> |
raise CrabException('Could not create tar-ball '+self.tgzNameWithPath) |
| 806 |
|
|
| 807 |
|
## check for tarball size |
| 808 |
|
tarballinfo = os.stat(self.tgzNameWithPath) |
| 810 |
|
raise CrabException('Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' + str(self.MaxTarBallSize) + ' MB input sandbox limit and not supported by the used GRID submission system. Please make sure that no unnecessary files are in all data directories in your local CMSSW project area as they are automatically packed into the input sandbox.') |
| 811 |
|
|
| 812 |
|
## create tar-ball with ML stuff |
| 786 |
– |
self.MLtgzfile = common.work_space.pathForTgz()+'share/MLfiles.tgz' |
| 787 |
– |
try: |
| 788 |
– |
tar = tarfile.open(self.MLtgzfile, "w:gz") |
| 789 |
– |
path=os.environ['CRABDIR'] + '/python/' |
| 790 |
– |
#for file in ['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py', 'parseCrabFjr.py','writeCfg.py']: |
| 791 |
– |
### FEDE #### |
| 792 |
– |
for file in ['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py', 'parseCrabFjr.py','writeCfg.py', 'JobReportErrorCode.py']: |
| 793 |
– |
############### |
| 794 |
– |
tar.add(path+file,file) |
| 795 |
– |
common.logger.debug(5,"Files added to "+self.MLtgzfile+" : "+str(tar.getnames())) |
| 796 |
– |
tar.close() |
| 797 |
– |
except : |
| 798 |
– |
raise CrabException('Could not create ML files tar-ball') |
| 799 |
– |
|
| 800 |
– |
return |
| 801 |
– |
|
| 802 |
– |
def additionalInputFileTgz(self): |
| 803 |
– |
""" |
| 804 |
– |
Put all additional files into a tar ball and return its name |
| 805 |
– |
""" |
| 806 |
– |
import tarfile |
| 807 |
– |
tarName= common.work_space.pathForTgz()+'share/'+self.additional_tgz_name |
| 808 |
– |
tar = tarfile.open(tarName, "w:gz") |
| 809 |
– |
for file in self.additional_inbox_files: |
| 810 |
– |
tar.add(file,string.split(file,'/')[-1]) |
| 811 |
– |
common.logger.debug(5,"Files added to "+self.additional_tgz_name+" : "+str(tar.getnames())) |
| 812 |
– |
tar.close() |
| 813 |
– |
return tarName |
| 813 |
|
|
| 814 |
|
def wsSetupEnvironment(self, nj=0): |
| 815 |
|
""" |
| 816 |
|
Returns part of a job script which prepares |
| 817 |
|
the execution environment for the job 'nj'. |
| 818 |
|
""" |
| 819 |
+ |
if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3): |
| 820 |
+ |
psetName = 'pset.py' |
| 821 |
+ |
else: |
| 822 |
+ |
psetName = 'pset.cfg' |
| 823 |
|
# Prepare JobType-independent part |
| 824 |
|
txt = '\n#Written by cms_cmssw::wsSetupEnvironment\n' |
| 825 |
|
txt += 'echo ">>> setup environment"\n' |
| 828 |
|
txt += 'elif [ $middleware == OSG ]; then\n' |
| 829 |
|
txt += ' WORKING_DIR=`/bin/mktemp -d $OSG_WN_TMP/cms_XXXXXXXXXXXX`\n' |
| 830 |
|
txt += ' if [ ! $? == 0 ] ;then\n' |
| 828 |
– |
#txt += ' echo "SET_CMS_ENV 10016 ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n' |
| 829 |
– |
#txt += ' echo "JOB_EXIT_STATUS = 10016"\n' |
| 830 |
– |
#txt += ' echo "JobExitCode=10016" | tee -a $RUNTIME_AREA/$repo\n' |
| 831 |
– |
#txt += ' dumpStatus $RUNTIME_AREA/$repo\n' |
| 832 |
– |
#txt += ' exit 1\n' |
| 831 |
|
txt += ' echo "ERROR ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n' |
| 832 |
|
txt += ' job_exit_code=10016\n' |
| 833 |
|
txt += ' func_exit\n' |
| 838 |
|
txt += ' cd $WORKING_DIR\n' |
| 839 |
|
txt += ' echo ">>> current directory (WORKING_DIR): $WORKING_DIR"\n' |
| 840 |
|
txt += self.wsSetupCMSOSGEnvironment_() |
| 843 |
– |
#txt += ' echo "### Set SCRAM ARCH to ' + self.executable_arch + ' ###"\n' |
| 844 |
– |
#txt += ' export SCRAM_ARCH='+self.executable_arch+'\n' |
| 841 |
|
txt += 'fi\n' |
| 842 |
|
|
| 843 |
|
# Prepare JobType-specific part |
| 848 |
|
txt += scram+' project CMSSW '+self.version+'\n' |
| 849 |
|
txt += 'status=$?\n' |
| 850 |
|
txt += 'if [ $status != 0 ] ; then\n' |
| 855 |
– |
#txt += ' echo "SET_EXE_ENV 10034 ==>ERROR CMSSW '+self.version+' not found on `hostname`" \n' |
| 856 |
– |
#txt += ' echo "JOB_EXIT_STATUS = 10034"\n' |
| 857 |
– |
#txt += ' echo "JobExitCode=10034" | tee -a $RUNTIME_AREA/$repo\n' |
| 858 |
– |
#txt += ' dumpStatus $RUNTIME_AREA/$repo\n' |
| 851 |
|
txt += ' echo "ERROR ==> CMSSW '+self.version+' not found on `hostname`" \n' |
| 852 |
|
txt += ' job_exit_code=10034\n' |
| 861 |
– |
#txt += ' if [ $middleware == OSG ]; then \n' |
| 862 |
– |
#txt += ' cd $RUNTIME_AREA\n' |
| 863 |
– |
#txt += ' echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n' |
| 864 |
– |
#txt += ' echo ">>> Remove working directory: $WORKING_DIR"\n' |
| 865 |
– |
#txt += ' /bin/rm -rf $WORKING_DIR\n' |
| 866 |
– |
#txt += ' if [ -d $WORKING_DIR ] ;then\n' |
| 867 |
– |
#txt += ' echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n' |
| 868 |
– |
#txt += ' echo "JOB_EXIT_STATUS = 10018"\n' |
| 869 |
– |
#txt += ' echo "JobExitCode=10018" | tee -a $RUNTIME_AREA/$repo\n' |
| 870 |
– |
#txt += ' dumpStatus $RUNTIME_AREA/$repo\n' |
| 871 |
– |
#txt += ' echo "ERROR ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n' |
| 872 |
– |
#txt += ' job_exit_code=10017\n' |
| 873 |
– |
#txt += ' fi\n' |
| 874 |
– |
#txt += ' fi \n' |
| 875 |
– |
#txt += ' exit 1 \n' |
| 853 |
|
txt += ' func_exit\n' |
| 854 |
|
txt += 'fi \n' |
| 855 |
|
txt += 'cd '+self.version+'\n' |
| 879 |
– |
########## FEDE FOR DBS2 ###################### |
| 856 |
|
txt += 'SOFTWARE_DIR=`pwd`\n' |
| 857 |
|
txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n' |
| 882 |
– |
############################################### |
| 883 |
– |
### needed grep for bug in scramv1 ### |
| 858 |
|
txt += 'eval `'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME`\n' |
| 859 |
+ |
txt += 'if [ $? != 0 ] ; then\n' |
| 860 |
+ |
txt += ' echo "ERROR ==> Problem with the command: "\n' |
| 861 |
+ |
txt += ' echo "eval \`'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME \` at `hostname`"\n' |
| 862 |
+ |
txt += ' job_exit_code=10034\n' |
| 863 |
+ |
txt += ' func_exit\n' |
| 864 |
+ |
txt += 'fi \n' |
| 865 |
|
# Handle the arguments: |
| 866 |
|
txt += "\n" |
| 867 |
|
txt += "## number of arguments (first argument always jobnumber)\n" |
| 868 |
|
txt += "\n" |
| 889 |
– |
# txt += "if [ $nargs -lt "+str(len(self.argsList[nj].split()))+" ]\n" |
| 869 |
|
txt += "if [ $nargs -lt "+str(self.argsList)+" ]\n" |
| 870 |
|
txt += "then\n" |
| 892 |
– |
#txt += " echo 'SET_EXE_ENV 1 ==> ERROR Too few arguments' +$nargs+ \n" |
| 893 |
– |
#txt += ' echo "JOB_EXIT_STATUS = 50113"\n' |
| 894 |
– |
#txt += ' echo "JobExitCode=50113" | tee -a $RUNTIME_AREA/$repo\n' |
| 895 |
– |
#txt += ' dumpStatus $RUNTIME_AREA/$repo\n' |
| 871 |
|
txt += " echo 'ERROR ==> Too few arguments' +$nargs+ \n" |
| 872 |
|
txt += ' job_exit_code=50113\n' |
| 898 |
– |
#txt += ' if [ $middleware == OSG ]; then \n' |
| 899 |
– |
#txt += ' cd $RUNTIME_AREA\n' |
| 900 |
– |
#txt += ' echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n' |
| 901 |
– |
#txt += ' echo ">>> Remove working directory: $WORKING_DIR"\n' |
| 902 |
– |
#txt += ' /bin/rm -rf $WORKING_DIR\n' |
| 903 |
– |
#txt += ' if [ -d $WORKING_DIR ] ;then\n' |
| 904 |
– |
#txt += ' echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n' |
| 905 |
– |
#txt += ' echo "JOB_EXIT_STATUS = 50114"\n' |
| 906 |
– |
#txt += ' echo "JobExitCode=50114" | tee -a $RUNTIME_AREA/$repo\n' |
| 907 |
– |
#txt += ' dumpStatus $RUNTIME_AREA/$repo\n' |
| 908 |
– |
#txt += ' echo "ERROR ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n' |
| 909 |
– |
#txt += ' job_exit_code=10017\n' |
| 910 |
– |
#txt += ' fi\n' |
| 911 |
– |
#txt += ' fi\n' |
| 912 |
– |
#txt += " exit 1\n" |
| 873 |
|
txt += " func_exit\n" |
| 874 |
|
txt += "fi\n" |
| 875 |
|
txt += "\n" |
| 876 |
|
|
| 877 |
|
# Prepare job-specific part |
| 878 |
|
job = common.job_list[nj] |
| 919 |
– |
### FEDE FOR DBS OUTPUT PUBLICATION |
| 879 |
|
if (self.datasetPath): |
| 880 |
|
txt += '\n' |
| 881 |
|
txt += 'DatasetPath='+self.datasetPath+'\n' |
| 882 |
|
|
| 883 |
|
datasetpath_split = self.datasetPath.split("/") |
| 884 |
< |
|
| 884 |
> |
### FEDE FOR NEW LFN ### |
| 885 |
> |
self.primaryDataset = datasetpath_split[1] |
| 886 |
> |
######################## |
| 887 |
|
txt += 'PrimaryDataset='+datasetpath_split[1]+'\n' |
| 888 |
|
txt += 'DataTier='+datasetpath_split[2]+'\n' |
| 889 |
|
txt += 'ApplicationFamily=cmsRun\n' |
| 890 |
|
|
| 891 |
|
else: |
| 892 |
|
txt += 'DatasetPath=MCDataTier\n' |
| 893 |
+ |
### FEDE FOR NEW LFN ### |
| 894 |
+ |
self.primaryDataset = 'null' |
| 895 |
+ |
######################## |
| 896 |
|
txt += 'PrimaryDataset=null\n' |
| 897 |
|
txt += 'DataTier=null\n' |
| 898 |
|
txt += 'ApplicationFamily=MCDataTier\n' |
| 899 |
< |
if self.pset != None: |
| 899 |
> |
if self.pset != None: |
| 900 |
|
pset = os.path.basename(job.configFilename()) |
| 901 |
|
txt += '\n' |
| 902 |
|
txt += 'cp $RUNTIME_AREA/'+pset+' .\n' |
| 903 |
|
if (self.datasetPath): # standard job |
| 904 |
|
txt += 'InputFiles=${args[1]}; export InputFiles\n' |
| 905 |
< |
txt += 'MaxEvents=${args[2]}; export MaxEvents\n' |
| 906 |
< |
txt += 'SkipEvents=${args[3]}; export SkipEvents\n' |
| 905 |
> |
if (self.useParent): |
| 906 |
> |
txt += 'ParentFiles=${args[2]}; export ParentFiles\n' |
| 907 |
> |
txt += 'MaxEvents=${args[3]}; export MaxEvents\n' |
| 908 |
> |
txt += 'SkipEvents=${args[4]}; export SkipEvents\n' |
| 909 |
> |
else: |
| 910 |
> |
txt += 'MaxEvents=${args[2]}; export MaxEvents\n' |
| 911 |
> |
txt += 'SkipEvents=${args[3]}; export SkipEvents\n' |
| 912 |
|
txt += 'echo "Inputfiles:<$InputFiles>"\n' |
| 913 |
+ |
if (self.useParent): txt += 'echo "ParentFiles:<$ParentFiles>"\n' |
| 914 |
|
txt += 'echo "MaxEvents:<$MaxEvents>"\n' |
| 915 |
|
txt += 'echo "SkipEvents:<$SkipEvents>"\n' |
| 916 |
|
else: # pythia like job |
| 922 |
|
txt += 'FirstRun=${args[1]}; export FirstRun\n' |
| 923 |
|
txt += 'echo "FirstRun: <$FirstRun>"\n' |
| 924 |
|
|
| 925 |
< |
txt += 'mv -f '+pset+' pset.cfg\n' |
| 925 |
> |
txt += 'mv -f ' + pset + ' ' + psetName + '\n' |
| 926 |
|
|
| 957 |
– |
if len(self.additional_inbox_files) > 0: |
| 958 |
– |
txt += 'if [ -e $RUNTIME_AREA/'+self.additional_tgz_name+' ] ; then\n' |
| 959 |
– |
txt += ' tar xzvf $RUNTIME_AREA/'+self.additional_tgz_name+'\n' |
| 960 |
– |
txt += 'fi\n' |
| 961 |
– |
pass |
| 927 |
|
|
| 928 |
|
if self.pset != None: |
| 929 |
+ |
# FUTURE: Can simply for 2_1_x and higher |
| 930 |
|
txt += '\n' |
| 931 |
< |
txt += 'echo "***** cat pset.cfg *********"\n' |
| 932 |
< |
txt += 'cat pset.cfg\n' |
| 933 |
< |
txt += 'echo "****** end pset.cfg ********"\n' |
| 934 |
< |
txt += '\n' |
| 935 |
< |
txt += 'PSETHASH=`EdmConfigHash < pset.cfg` \n' |
| 931 |
> |
if self.debug_wrapper==True: |
| 932 |
> |
txt += 'echo "***** cat ' + psetName + ' *********"\n' |
| 933 |
> |
txt += 'cat ' + psetName + '\n' |
| 934 |
> |
txt += 'echo "****** end ' + psetName + ' ********"\n' |
| 935 |
> |
txt += '\n' |
| 936 |
> |
if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3): |
| 937 |
> |
txt += 'PSETHASH=`edmConfigHash ' + psetName + '` \n' |
| 938 |
> |
else: |
| 939 |
> |
txt += 'PSETHASH=`edmConfigHash < ' + psetName + '` \n' |
| 940 |
|
txt += 'echo "PSETHASH = $PSETHASH" \n' |
| 941 |
|
txt += '\n' |
| 942 |
|
return txt |
| 943 |
< |
#### FEDE ##### |
| 943 |
> |
|
| 944 |
|
def wsUntarSoftware(self, nj=0): |
| 945 |
|
""" |
| 946 |
|
Put in the script the commands to build an executable |
| 952 |
|
if os.path.isfile(self.tgzNameWithPath): |
| 953 |
|
txt += 'echo ">>> tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+' :" \n' |
| 954 |
|
txt += 'tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n' |
| 955 |
+ |
if self.debug_wrapper: |
| 956 |
+ |
txt += 'ls -Al \n' |
| 957 |
|
txt += 'untar_status=$? \n' |
| 958 |
|
txt += 'if [ $untar_status -ne 0 ]; then \n' |
| 959 |
|
txt += ' echo "ERROR ==> Untarring .tgz file failed"\n' |
| 963 |
|
txt += ' echo "Successful untar" \n' |
| 964 |
|
txt += 'fi \n' |
| 965 |
|
txt += '\n' |
| 966 |
< |
txt += 'echo ">>> Include ProdCommon in PYTHONPATH:"\n' |
| 966 |
> |
txt += 'echo ">>> Include $RUNTIME_AREA in PYTHONPATH:"\n' |
| 967 |
|
txt += 'if [ -z "$PYTHONPATH" ]; then\n' |
| 968 |
< |
txt += ' export PYTHONPATH=$RUNTIME_AREA/ProdCommon\n' |
| 968 |
> |
txt += ' export PYTHONPATH=$RUNTIME_AREA/\n' |
| 969 |
|
txt += 'else\n' |
| 970 |
< |
txt += ' export PYTHONPATH=$RUNTIME_AREA/ProdCommon:${PYTHONPATH}\n' |
| 970 |
> |
txt += ' export PYTHONPATH=$RUNTIME_AREA/:${PYTHONPATH}\n' |
| 971 |
|
txt += 'echo "PYTHONPATH=$PYTHONPATH"\n' |
| 972 |
|
txt += 'fi\n' |
| 973 |
|
txt += '\n' |
| 975 |
|
pass |
| 976 |
|
|
| 977 |
|
return txt |
| 978 |
< |
|
| 978 |
> |
|
| 979 |
|
def wsBuildExe(self, nj=0): |
| 980 |
|
""" |
| 981 |
|
Put in the script the commands to build an executable |
| 985 |
|
txt = '\n#Written by cms_cmssw::wsBuildExe\n' |
| 986 |
|
txt += 'echo ">>> moving CMSSW software directories in `pwd`" \n' |
| 987 |
|
|
| 988 |
< |
txt += 'mv $RUNTIME_AREA/lib . \n' |
| 989 |
< |
txt += 'mv $RUNTIME_AREA/module . \n' |
| 990 |
< |
txt += 'mv $RUNTIME_AREA/ProdCommon . \n' |
| 991 |
< |
|
| 992 |
< |
|
| 993 |
< |
#if os.path.isfile(self.tgzNameWithPath): |
| 994 |
< |
# txt += 'echo ">>> tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+' :" \n' |
| 995 |
< |
# txt += 'tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n' |
| 996 |
< |
# txt += 'untar_status=$? \n' |
| 997 |
< |
# txt += 'if [ $untar_status -ne 0 ]; then \n' |
| 998 |
< |
# txt += ' echo "ERROR ==> Untarring .tgz file failed"\n' |
| 999 |
< |
# txt += ' job_exit_code=$untar_status\n' |
| 1000 |
< |
# txt += ' func_exit\n' |
| 1029 |
< |
# txt += 'else \n' |
| 1030 |
< |
# txt += ' echo "Successful untar" \n' |
| 1031 |
< |
# txt += 'fi \n' |
| 1032 |
< |
# txt += '\n' |
| 1033 |
< |
# txt += 'echo ">>> Include ProdCommon in PYTHONPATH:"\n' |
| 988 |
> |
txt += 'rm -r lib/ module/ \n' |
| 989 |
> |
txt += 'mv $RUNTIME_AREA/lib/ . \n' |
| 990 |
> |
txt += 'mv $RUNTIME_AREA/module/ . \n' |
| 991 |
> |
if self.dataExist == True: |
| 992 |
> |
txt += 'rm -r src/ \n' |
| 993 |
> |
txt += 'mv $RUNTIME_AREA/src/ . \n' |
| 994 |
> |
if len(self.additional_inbox_files)>0: |
| 995 |
> |
for file in self.additional_inbox_files: |
| 996 |
> |
txt += 'mv $RUNTIME_AREA/'+os.path.basename(file)+' . \n' |
| 997 |
> |
# txt += 'mv $RUNTIME_AREA/ProdCommon/ . \n' |
| 998 |
> |
# txt += 'mv $RUNTIME_AREA/IMProv/ . \n' |
| 999 |
> |
|
| 1000 |
> |
txt += 'echo ">>> Include $RUNTIME_AREA in PYTHONPATH:"\n' |
| 1001 |
|
txt += 'if [ -z "$PYTHONPATH" ]; then\n' |
| 1002 |
< |
txt += ' export PYTHONPATH=$SOFTWARE_DIR/ProdCommon\n' |
| 1002 |
> |
txt += ' export PYTHONPATH=$RUNTIME_AREA/\n' |
| 1003 |
|
txt += 'else\n' |
| 1004 |
< |
txt += ' export PYTHONPATH=$SOFTWARE_DIR/ProdCommon:${PYTHONPATH}\n' |
| 1004 |
> |
txt += ' export PYTHONPATH=$RUNTIME_AREA/:${PYTHONPATH}\n' |
| 1005 |
|
txt += 'echo "PYTHONPATH=$PYTHONPATH"\n' |
| 1006 |
|
txt += 'fi\n' |
| 1007 |
|
txt += '\n' |
| 1008 |
|
|
| 1009 |
|
return txt |
| 1043 |
– |
############################################################################ |
| 1010 |
|
|
| 1045 |
– |
def modifySteeringCards(self, nj): |
| 1046 |
– |
""" |
| 1047 |
– |
modify the card provided by the user, |
| 1048 |
– |
writing a new card into share dir |
| 1049 |
– |
""" |
| 1011 |
|
|
| 1012 |
|
def executableName(self): |
| 1013 |
< |
if self.scriptExe: #CarlosDaniele |
| 1013 |
> |
if self.scriptExe: |
| 1014 |
|
return "sh " |
| 1015 |
|
else: |
| 1016 |
|
return self.executable |
| 1020 |
|
if self.scriptExe:#CarlosDaniele |
| 1021 |
|
return self.scriptExe + " $NJob" |
| 1022 |
|
else: |
| 1062 |
– |
version_array = self.scram.getSWVersion().split('_') |
| 1063 |
– |
major = 0 |
| 1064 |
– |
minor = 0 |
| 1065 |
– |
try: |
| 1066 |
– |
major = int(version_array[1]) |
| 1067 |
– |
minor = int(version_array[2]) |
| 1068 |
– |
except: |
| 1069 |
– |
msg = "Cannot parse CMSSW version string: " + "_".join(version_array) + " for major and minor release number!" |
| 1070 |
– |
raise CrabException(msg) |
| 1071 |
– |
|
| 1023 |
|
ex_args = "" |
| 1024 |
< |
|
| 1024 |
> |
# FUTURE: This tests the CMSSW version. Can remove code as versions deprecated |
| 1025 |
|
# Framework job report |
| 1026 |
< |
if major >= 1 and minor >= 5 : |
| 1076 |
< |
#ex_args += " -j " + self.fjrFileName |
| 1077 |
< |
### FEDE it could be improved!!! #### |
| 1026 |
> |
if (self.CMSSW_major >= 1 and self.CMSSW_minor >= 5) or (self.CMSSW_major >= 2): |
| 1027 |
|
ex_args += " -j $RUNTIME_AREA/crab_fjr_$NJob.xml" |
| 1028 |
< |
####################################### |
| 1029 |
< |
# Type of cfg file |
| 1030 |
< |
if major >= 2 : |
| 1082 |
< |
ex_args += " -p pset.pycfg" |
| 1028 |
> |
# Type of config file |
| 1029 |
> |
if self.CMSSW_major >= 2 : |
| 1030 |
> |
ex_args += " -p pset.py" |
| 1031 |
|
else: |
| 1032 |
|
ex_args += " -p pset.cfg" |
| 1033 |
|
return ex_args |
| 1037 |
|
Returns a list of filenames to be put in JDL input sandbox. |
| 1038 |
|
""" |
| 1039 |
|
inp_box = [] |
| 1092 |
– |
# # dict added to delete duplicate from input sandbox file list |
| 1093 |
– |
# seen = {} |
| 1094 |
– |
## code |
| 1040 |
|
if os.path.isfile(self.tgzNameWithPath): |
| 1041 |
|
inp_box.append(self.tgzNameWithPath) |
| 1097 |
– |
if os.path.isfile(self.MLtgzfile): |
| 1098 |
– |
inp_box.append(self.MLtgzfile) |
| 1099 |
– |
## config |
| 1100 |
– |
if not self.pset is None: |
| 1101 |
– |
inp_box.append(common.work_space.pathForTgz() + 'job/' + self.configFilename()) |
| 1102 |
– |
## additional input files |
| 1103 |
– |
tgz = self.additionalInputFileTgz() |
| 1104 |
– |
inp_box.append(tgz) |
| 1105 |
– |
## executable |
| 1042 |
|
wrapper = os.path.basename(str(common._db.queryTask('scriptName'))) |
| 1043 |
|
inp_box.append(common.work_space.pathForTgz() +'job/'+ wrapper) |
| 1044 |
|
return inp_box |
| 1052 |
|
## User Declared output files |
| 1053 |
|
for out in (self.output_file+self.output_file_sandbox): |
| 1054 |
|
n_out = nj + 1 |
| 1055 |
< |
out_box.append(self.numberFile_(out,str(n_out))) |
| 1055 |
> |
out_box.append(numberFile(out,str(n_out))) |
| 1056 |
|
return out_box |
| 1057 |
|
|
| 1122 |
– |
def prepareSteeringCards(self): |
| 1123 |
– |
""" |
| 1124 |
– |
Make initial modifications of the user's steering card file. |
| 1125 |
– |
""" |
| 1126 |
– |
return |
| 1058 |
|
|
| 1059 |
|
def wsRenameOutput(self, nj): |
| 1060 |
|
""" |
| 1064 |
|
txt = '\n#Written by cms_cmssw::wsRenameOutput\n' |
| 1065 |
|
txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n' |
| 1066 |
|
txt += 'echo ">>> current directory content:"\n' |
| 1067 |
< |
txt += 'ls \n' |
| 1067 |
> |
if self.debug_wrapper: |
| 1068 |
> |
txt += 'ls -Al\n' |
| 1069 |
|
txt += '\n' |
| 1070 |
|
|
| 1139 |
– |
#txt += 'output_exit_status=0\n' |
| 1140 |
– |
|
| 1141 |
– |
### FEDE ####### |
| 1142 |
– |
#for fileWithSuffix in (self.output_file_sandbox): |
| 1143 |
– |
# output_file_num = self.numberFile_(fileWithSuffix, '$NJob') |
| 1144 |
– |
# txt += '\n' |
| 1145 |
– |
# txt += '# check output file\n' |
| 1146 |
– |
# txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n' |
| 1147 |
– |
# txt += ' mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n' |
| 1148 |
– |
# txt += ' ln -s $RUNTIME_AREA/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n' |
| 1149 |
– |
# txt += 'else\n' |
| 1150 |
– |
# txt += ' echo "WARNING: Output file '+fileWithSuffix+' not found"\n' |
| 1151 |
– |
# txt += ' job_exit_code=60302\n' |
| 1152 |
– |
# if common.scheduler.name().upper() == 'CONDOR_G': |
| 1153 |
– |
# txt += ' if [ $middleware == OSG ]; then \n' |
| 1154 |
– |
# txt += ' echo "prepare dummy output file"\n' |
| 1155 |
– |
# txt += ' echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n' |
| 1156 |
– |
# txt += ' fi \n' |
| 1157 |
– |
# txt += 'fi\n' |
| 1158 |
– |
|
| 1071 |
|
for fileWithSuffix in (self.output_file): |
| 1072 |
< |
output_file_num = self.numberFile_(fileWithSuffix, '$NJob') |
| 1072 |
> |
output_file_num = numberFile(fileWithSuffix, '$NJob') |
| 1073 |
|
txt += '\n' |
| 1074 |
|
txt += '# check output file\n' |
| 1075 |
|
txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n' |
| 1080 |
|
txt += ' mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n' |
| 1081 |
|
txt += ' ln -s $RUNTIME_AREA/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n' |
| 1082 |
|
txt += 'else\n' |
| 1171 |
– |
#txt += ' exit_status=60302\n' |
| 1172 |
– |
#txt += ' echo "ERROR: Output file '+fileWithSuffix+' not found"\n' |
| 1173 |
– |
#txt += ' echo "JOB_EXIT_STATUS = $exit_status"\n' |
| 1174 |
– |
#txt += ' output_exit_status=$exit_status\n' |
| 1083 |
|
txt += ' job_exit_code=60302\n' |
| 1084 |
|
txt += ' echo "WARNING: Output file '+fileWithSuffix+' not found"\n' |
| 1085 |
|
if common.scheduler.name().upper() == 'CONDOR_G': |
| 1090 |
|
txt += 'fi\n' |
| 1091 |
|
file_list = [] |
| 1092 |
|
for fileWithSuffix in (self.output_file): |
| 1093 |
< |
file_list.append(self.numberFile_(fileWithSuffix, '$NJob')) |
| 1093 |
> |
file_list.append(numberFile(fileWithSuffix, '$NJob')) |
| 1094 |
|
|
| 1095 |
|
txt += 'file_list="'+string.join(file_list,' ')+'"\n' |
| 1096 |
|
txt += '\n' |
| 1097 |
|
txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n' |
| 1098 |
|
txt += 'echo ">>> current directory content:"\n' |
| 1099 |
< |
txt += 'ls \n' |
| 1099 |
> |
if self.debug_wrapper: |
| 1100 |
> |
txt += 'ls -Al\n' |
| 1101 |
|
txt += '\n' |
| 1102 |
|
txt += 'cd $RUNTIME_AREA\n' |
| 1103 |
|
txt += 'echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n' |
| 1104 |
|
return txt |
| 1105 |
|
|
| 1197 |
– |
def numberFile_(self, file, txt): |
| 1198 |
– |
""" |
| 1199 |
– |
append _'txt' before last extension of a file |
| 1200 |
– |
""" |
| 1201 |
– |
p = string.split(file,".") |
| 1202 |
– |
# take away last extension |
| 1203 |
– |
name = p[0] |
| 1204 |
– |
for x in p[1:-1]: |
| 1205 |
– |
name=name+"."+x |
| 1206 |
– |
# add "_txt" |
| 1207 |
– |
if len(p)>1: |
| 1208 |
– |
ext = p[len(p)-1] |
| 1209 |
– |
result = name + '_' + txt + "." + ext |
| 1210 |
– |
else: |
| 1211 |
– |
result = name + '_' + txt |
| 1212 |
– |
|
| 1213 |
– |
return result |
| 1214 |
– |
|
| 1106 |
|
def getRequirements(self, nj=[]): |
| 1107 |
|
""" |
| 1108 |
|
return job requirements to add to jdl files |
| 1112 |
|
req='Member("VO-cms-' + \ |
| 1113 |
|
self.version + \ |
| 1114 |
|
'", other.GlueHostApplicationSoftwareRunTimeEnvironment)' |
| 1115 |
< |
## SL add requirement for OS version only if SL4 |
| 1225 |
< |
#reSL4 = re.compile( r'slc4' ) |
| 1226 |
< |
if self.executable_arch: # and reSL4.search(self.executable_arch): |
| 1115 |
> |
if self.executable_arch: |
| 1116 |
|
req+=' && Member("VO-cms-' + \ |
| 1117 |
|
self.executable_arch + \ |
| 1118 |
|
'", other.GlueHostApplicationSoftwareRunTimeEnvironment)' |
| 1119 |
|
|
| 1120 |
|
req = req + ' && (other.GlueHostNetworkAdapterOutboundIP)' |
| 1121 |
< |
if common.scheduler.name() == "glitecoll": |
| 1121 |
> |
if ( common.scheduler.name() == "glitecoll" ) or ( common.scheduler.name() == "glite"): |
| 1122 |
|
req += ' && other.GlueCEStateStatus == "Production" ' |
| 1123 |
|
|
| 1124 |
|
return req |
| 1125 |
|
|
| 1126 |
|
def configFilename(self): |
| 1127 |
|
""" return the config filename """ |
| 1128 |
< |
return self.name()+'.cfg' |
| 1128 |
> |
# FUTURE: Can remove cfg mode for CMSSW >= 2_1_x |
| 1129 |
> |
if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3): |
| 1130 |
> |
return self.name()+'.py' |
| 1131 |
> |
else: |
| 1132 |
> |
return self.name()+'.cfg' |
| 1133 |
|
|
| 1134 |
|
def wsSetupCMSOSGEnvironment_(self): |
| 1135 |
|
""" |
| 1145 |
|
txt += ' # Use $OSG_APP/cmssoft/cms/cmsset_default.sh to setup cms software\n' |
| 1146 |
|
txt += ' source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n' |
| 1147 |
|
txt += ' else\n' |
| 1255 |
– |
#txt += ' echo "SET_CMS_ENV 10020 ==> ERROR $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n' |
| 1256 |
– |
#txt += ' echo "JOB_EXIT_STATUS = 10020"\n' |
| 1257 |
– |
#txt += ' echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n' |
| 1258 |
– |
#txt += ' dumpStatus $RUNTIME_AREA/$repo\n' |
| 1148 |
|
txt += ' echo "ERROR ==> $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n' |
| 1149 |
|
txt += ' job_exit_code=10020\n' |
| 1261 |
– |
#txt += ' cd $RUNTIME_AREA\n' |
| 1262 |
– |
#txt += ' echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n' |
| 1263 |
– |
#txt += ' echo ">>> Remove working directory: $WORKING_DIR"\n' |
| 1264 |
– |
#txt += ' /bin/rm -rf $WORKING_DIR\n' |
| 1265 |
– |
#txt += ' if [ -d $WORKING_DIR ] ;then\n' |
| 1266 |
– |
#txt += ' echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n' |
| 1267 |
– |
#txt += ' echo "JOB_EXIT_STATUS = 10017"\n' |
| 1268 |
– |
#txt += ' echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n' |
| 1269 |
– |
#txt += ' dumpStatus $RUNTIME_AREA/$repo\n' |
| 1270 |
– |
#txt += ' echo "ERROR ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n' |
| 1271 |
– |
#txt += ' job_exit_code=10017\n' |
| 1272 |
– |
#txt += ' fi\n' |
| 1273 |
– |
txt += '\n' |
| 1274 |
– |
#txt += ' exit 1\n' |
| 1150 |
|
txt += ' func_exit\n' |
| 1151 |
|
txt += ' fi\n' |
| 1152 |
|
txt += '\n' |
| 1155 |
|
|
| 1156 |
|
return txt |
| 1157 |
|
|
| 1283 |
– |
### OLI_DANIELE |
| 1158 |
|
def wsSetupCMSLCGEnvironment_(self): |
| 1159 |
|
""" |
| 1160 |
|
Returns part of a job script which is prepares |
| 1166 |
|
txt += ' export SCRAM_ARCH='+self.executable_arch+'\n' |
| 1167 |
|
txt += ' export BUILD_ARCH='+self.executable_arch+'\n' |
| 1168 |
|
txt += ' if [ ! $VO_CMS_SW_DIR ] ;then\n' |
| 1295 |
– |
#txt += ' echo "SET_CMS_ENV 10031 ==> ERROR CMS software dir not found on WN `hostname`"\n' |
| 1296 |
– |
#txt += ' echo "JOB_EXIT_STATUS = 10031" \n' |
| 1297 |
– |
#txt += ' echo "JobExitCode=10031" | tee -a $RUNTIME_AREA/$repo\n' |
| 1298 |
– |
#txt += ' dumpStatus $RUNTIME_AREA/$repo\n' |
| 1299 |
– |
#txt += ' exit 1\n' |
| 1169 |
|
txt += ' echo "ERROR ==> CMS software dir not found on WN `hostname`"\n' |
| 1170 |
|
txt += ' job_exit_code=10031\n' |
| 1171 |
|
txt += ' func_exit\n' |
| 1172 |
|
txt += ' else\n' |
| 1173 |
|
txt += ' echo "Sourcing environment... "\n' |
| 1174 |
|
txt += ' if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n' |
| 1306 |
– |
#txt += ' echo "SET_CMS_ENV 10020 ==> ERROR cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n' |
| 1307 |
– |
#txt += ' echo "JOB_EXIT_STATUS = 10020"\n' |
| 1308 |
– |
#txt += ' echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n' |
| 1309 |
– |
#txt += ' dumpStatus $RUNTIME_AREA/$repo\n' |
| 1310 |
– |
#txt += ' exit 1\n' |
| 1175 |
|
txt += ' echo "ERROR ==> cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n' |
| 1176 |
|
txt += ' job_exit_code=10020\n' |
| 1177 |
|
txt += ' func_exit\n' |
| 1180 |
|
txt += ' source $VO_CMS_SW_DIR/cmsset_default.sh\n' |
| 1181 |
|
txt += ' result=$?\n' |
| 1182 |
|
txt += ' if [ $result -ne 0 ]; then\n' |
| 1319 |
– |
#txt += ' echo "SET_CMS_ENV 10032 ==> ERROR problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n' |
| 1320 |
– |
#txt += ' echo "JOB_EXIT_STATUS = 10032"\n' |
| 1321 |
– |
#txt += ' echo "JobExitCode=10032" | tee -a $RUNTIME_AREA/$repo\n' |
| 1322 |
– |
#txt += ' dumpStatus $RUNTIME_AREA/$repo\n' |
| 1323 |
– |
#txt += ' exit 1\n' |
| 1183 |
|
txt += ' echo "ERROR ==> problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n' |
| 1184 |
|
txt += ' job_exit_code=10032\n' |
| 1185 |
|
txt += ' func_exit\n' |
| 1189 |
|
txt += ' echo "==> setup cms environment ok"\n' |
| 1190 |
|
return txt |
| 1191 |
|
|
| 1333 |
– |
### FEDE FOR DBS OUTPUT PUBLICATION |
| 1192 |
|
def modifyReport(self, nj): |
| 1193 |
|
""" |
| 1194 |
|
insert the part of the script that modifies the FrameworkJob Report |
| 1195 |
|
""" |
| 1338 |
– |
|
| 1196 |
|
txt = '\n#Written by cms_cmssw::modifyReport\n' |
| 1197 |
< |
try: |
| 1341 |
< |
publish_data = int(self.cfg_params['USER.publish_data']) |
| 1342 |
< |
except KeyError: |
| 1343 |
< |
publish_data = 0 |
| 1197 |
> |
publish_data = int(self.cfg_params.get('USER.publish_data',0)) |
| 1198 |
|
if (publish_data == 1): |
| 1199 |
< |
|
| 1199 |
> |
processedDataset = self.cfg_params['USER.publish_data_name'] |
| 1200 |
> |
if (self.primaryDataset == 'null'): |
| 1201 |
> |
self.primaryDataset = processedDataset |
| 1202 |
> |
if (common.scheduler.name().upper() == "CAF" or common.scheduler.name().upper() == "LSF"): |
| 1203 |
> |
### FEDE FOR NEW LFN ### |
| 1204 |
> |
LFNBaseName = LFNBase(self.primaryDataset, processedDataset, LocalUser=True) |
| 1205 |
> |
self.user = getUserName(LocalUser=True) |
| 1206 |
> |
######################## |
| 1207 |
> |
else : |
| 1208 |
> |
### FEDE FOR NEW LFN ### |
| 1209 |
> |
LFNBaseName = LFNBase(self.primaryDataset, processedDataset) |
| 1210 |
> |
self.user = getUserName() |
| 1211 |
> |
######################## |
| 1212 |
> |
|
| 1213 |
|
txt += 'if [ $copy_exit_status -eq 0 ]; then\n' |
| 1214 |
< |
txt += ' echo ">>> Modify Job Report:" \n' |
| 1215 |
< |
txt += ' chmod a+x $SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py\n' |
| 1216 |
< |
#txt += ' if [ -z "$SE" ]; then\n' |
| 1217 |
< |
#txt += ' SE="" \n' |
| 1218 |
< |
#txt += ' fi \n' |
| 1219 |
< |
#txt += ' if [ -z "$SE_PATH" ]; then\n' |
| 1220 |
< |
#txt += ' SE_PATH="" \n' |
| 1221 |
< |
#txt += ' fi \n' |
| 1222 |
< |
txt += ' echo "SE = $SE"\n' |
| 1356 |
< |
txt += ' echo "SE_PATH = $SE_PATH"\n' |
| 1214 |
> |
### FEDE FOR NEW LFN ### |
| 1215 |
> |
#txt += ' FOR_LFN=%s_${PSETHASH}/\n'%(LFNBaseName) |
| 1216 |
> |
txt += ' FOR_LFN=%s/${PSETHASH}/\n'%(LFNBaseName) |
| 1217 |
> |
######################## |
| 1218 |
> |
txt += 'else\n' |
| 1219 |
> |
txt += ' FOR_LFN=/copy_problems/ \n' |
| 1220 |
> |
txt += ' SE=""\n' |
| 1221 |
> |
txt += ' SE_PATH=""\n' |
| 1222 |
> |
txt += 'fi\n' |
| 1223 |
|
|
| 1224 |
< |
processedDataset = self.cfg_params['USER.publish_data_name'] |
| 1225 |
< |
txt += ' ProcessedDataset='+processedDataset+'\n' |
| 1226 |
< |
#txt += ' if [ "$SE_PATH" == "" ]; then\n' |
| 1227 |
< |
#txt += ' FOR_LFN=/copy_problems/ \n' |
| 1228 |
< |
#txt += ' else \n' |
| 1229 |
< |
#txt += ' tmp=`echo $SE_PATH | awk -F \'store\' \'{print$2}\'` \n' |
| 1230 |
< |
#txt += ' FOR_LFN=/store$tmp \n' |
| 1231 |
< |
#txt += ' fi \n' |
| 1232 |
< |
txt += ' tmp=`echo $SE_PATH | awk -F \'store\' \'{print$2}\'` \n' |
| 1233 |
< |
txt += ' FOR_LFN=/store$tmp \n' |
| 1234 |
< |
txt += ' echo "ProcessedDataset = $ProcessedDataset"\n' |
| 1235 |
< |
txt += ' echo "FOR_LFN = $FOR_LFN" \n' |
| 1236 |
< |
txt += ' echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n' |
| 1237 |
< |
#txt += ' echo "$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n' |
| 1238 |
< |
#txt += ' $SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n' |
| 1239 |
< |
### FEDE #### |
| 1240 |
< |
txt += ' echo "$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n' |
| 1241 |
< |
txt += ' $SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n' |
| 1242 |
< |
#################################### |
| 1243 |
< |
txt += ' modifyReport_result=$?\n' |
| 1378 |
< |
txt += ' if [ $modifyReport_result -ne 0 ]; then\n' |
| 1379 |
< |
txt += ' modifyReport_result=70500\n' |
| 1380 |
< |
txt += ' job_exit_code=$modifyReport_result\n' |
| 1381 |
< |
txt += ' echo "ModifyReportResult=$modifyReport_result" | tee -a $RUNTIME_AREA/$repo\n' |
| 1382 |
< |
txt += ' echo "WARNING: Problem with ModifyJobReport"\n' |
| 1383 |
< |
txt += ' else\n' |
| 1384 |
< |
### FEDE ##### |
| 1385 |
< |
#txt += ' mv NewFrameworkJobReport.xml crab_fjr_$NJob.xml\n' |
| 1386 |
< |
####################### |
| 1387 |
< |
txt += ' mv NewFrameworkJobReport.xml $RUNTIME_AREA/crab_fjr_$NJob.xml\n' |
| 1388 |
< |
txt += ' fi\n' |
| 1224 |
> |
txt += 'echo ">>> Modify Job Report:" \n' |
| 1225 |
> |
txt += 'chmod a+x $RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py\n' |
| 1226 |
> |
txt += 'ProcessedDataset='+processedDataset+'\n' |
| 1227 |
> |
txt += 'echo "ProcessedDataset = $ProcessedDataset"\n' |
| 1228 |
> |
txt += 'echo "SE = $SE"\n' |
| 1229 |
> |
txt += 'echo "SE_PATH = $SE_PATH"\n' |
| 1230 |
> |
txt += 'echo "FOR_LFN = $FOR_LFN" \n' |
| 1231 |
> |
txt += 'echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n' |
| 1232 |
> |
### FEDE FOR NEW LFN ### |
| 1233 |
> |
txt += 'echo "$RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier ' + self.user + '-$ProcessedDataset-$PSETHASH $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n' |
| 1234 |
> |
txt += '$RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier ' + self.user + '-$ProcessedDataset-$PSETHASH $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n' |
| 1235 |
> |
######################## |
| 1236 |
> |
txt += 'modifyReport_result=$?\n' |
| 1237 |
> |
txt += 'if [ $modifyReport_result -ne 0 ]; then\n' |
| 1238 |
> |
txt += ' modifyReport_result=70500\n' |
| 1239 |
> |
txt += ' job_exit_code=$modifyReport_result\n' |
| 1240 |
> |
txt += ' echo "ModifyReportResult=$modifyReport_result" | tee -a $RUNTIME_AREA/$repo\n' |
| 1241 |
> |
txt += ' echo "WARNING: Problem with ModifyJobReport"\n' |
| 1242 |
> |
txt += 'else\n' |
| 1243 |
> |
txt += ' mv NewFrameworkJobReport.xml $RUNTIME_AREA/crab_fjr_$NJob.xml\n' |
| 1244 |
|
txt += 'fi\n' |
| 1245 |
|
return txt |
| 1246 |
|
|
| 1247 |
< |
def cleanEnv(self): |
| 1248 |
< |
txt = '\n#Written by cms_cmssw::cleanEnv\n' |
| 1249 |
< |
txt += 'if [ $middleware == OSG ]; then\n' |
| 1250 |
< |
txt += ' cd $RUNTIME_AREA\n' |
| 1251 |
< |
txt += ' echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n' |
| 1252 |
< |
txt += ' echo ">>> Remove working directory: $WORKING_DIR"\n' |
| 1253 |
< |
txt += ' /bin/rm -rf $WORKING_DIR\n' |
| 1254 |
< |
txt += ' if [ -d $WORKING_DIR ] ;then\n' |
| 1255 |
< |
txt += ' echo "ERROR ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n' |
| 1256 |
< |
txt += ' job_exit_code=10017\n' |
| 1257 |
< |
txt += ' func_exit\n' |
| 1247 |
> |
def wsParseFJR(self): |
| 1248 |
> |
""" |
| 1249 |
> |
Parse the FrameworkJobReport to obtain useful infos |
| 1250 |
> |
""" |
| 1251 |
> |
txt = '\n#Written by cms_cmssw::wsParseFJR\n' |
| 1252 |
> |
txt += 'echo ">>> Parse FrameworkJobReport crab_fjr.xml"\n' |
| 1253 |
> |
txt += 'if [ -s $RUNTIME_AREA/crab_fjr_$NJob.xml ]; then\n' |
| 1254 |
> |
txt += ' if [ -s $RUNTIME_AREA/parseCrabFjr.py ]; then\n' |
| 1255 |
> |
txt += ' cmd_out=`python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --dashboard $MonitorID,$MonitorJobID '+self.debugWrap+'`\n' |
| 1256 |
> |
if self.debug_wrapper : |
| 1257 |
> |
txt += ' echo "Result of parsing the FrameworkJobReport crab_fjr.xml: $cmd_out"\n' |
| 1258 |
> |
txt += ' executable_exit_status=`python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --exitcode`\n' |
| 1259 |
> |
txt += ' if [ $executable_exit_status -eq 50115 ];then\n' |
| 1260 |
> |
txt += ' echo ">>> crab_fjr.xml contents: "\n' |
| 1261 |
> |
txt += ' cat $RUNTIME_AREA/crab_fjr_$NJob.xml\n' |
| 1262 |
> |
txt += ' echo "Wrong FrameworkJobReport --> does not contain useful info. ExitStatus: $executable_exit_status"\n' |
| 1263 |
> |
txt += ' elif [ $executable_exit_status -eq -999 ];then\n' |
| 1264 |
> |
txt += ' echo "ExitStatus from FrameworkJobReport not available. not available. Using exit code of executable from command line."\n' |
| 1265 |
> |
txt += ' else\n' |
| 1266 |
> |
txt += ' echo "Extracted ExitStatus from FrameworkJobReport parsing output: $executable_exit_status"\n' |
| 1267 |
> |
txt += ' fi\n' |
| 1268 |
> |
txt += ' else\n' |
| 1269 |
> |
txt += ' echo "CRAB python script to parse CRAB FrameworkJobReport crab_fjr.xml is not available, using exit code of executable from command line."\n' |
| 1270 |
|
txt += ' fi\n' |
| 1271 |
+ |
#### Patch to check input data reading for CMSSW16x Hopefully we-ll remove it asap |
| 1272 |
+ |
|
| 1273 |
+ |
txt += ' if [ $executable_exit_status -eq 0 ];then\n' |
| 1274 |
+ |
txt += ' echo ">>> Executable succeded $executable_exit_status"\n' |
| 1275 |
+ |
if (self.datasetPath and not (self.dataset_pu or self.useParent)) : |
| 1276 |
+ |
# VERIFY PROCESSED DATA |
| 1277 |
+ |
txt += ' echo ">>> Verify list of processed files:"\n' |
| 1278 |
+ |
txt += ' echo $InputFiles |tr -d \'\\\\\' |tr \',\' \'\\n\'|tr -d \'"\' > input-files.txt\n' |
| 1279 |
+ |
txt += ' python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --lfn > processed-files.txt\n' |
| 1280 |
+ |
txt += ' cat input-files.txt | sort | uniq > tmp.txt\n' |
| 1281 |
+ |
txt += ' mv tmp.txt input-files.txt\n' |
| 1282 |
+ |
txt += ' echo "cat input-files.txt"\n' |
| 1283 |
+ |
txt += ' echo "----------------------"\n' |
| 1284 |
+ |
txt += ' cat input-files.txt\n' |
| 1285 |
+ |
txt += ' cat processed-files.txt | sort | uniq > tmp.txt\n' |
| 1286 |
+ |
txt += ' mv tmp.txt processed-files.txt\n' |
| 1287 |
+ |
txt += ' echo "----------------------"\n' |
| 1288 |
+ |
txt += ' echo "cat processed-files.txt"\n' |
| 1289 |
+ |
txt += ' echo "----------------------"\n' |
| 1290 |
+ |
txt += ' cat processed-files.txt\n' |
| 1291 |
+ |
txt += ' echo "----------------------"\n' |
| 1292 |
+ |
txt += ' diff -q input-files.txt processed-files.txt\n' |
| 1293 |
+ |
txt += ' fileverify_status=$?\n' |
| 1294 |
+ |
txt += ' if [ $fileverify_status -ne 0 ]; then\n' |
| 1295 |
+ |
txt += ' executable_exit_status=30001\n' |
| 1296 |
+ |
txt += ' echo "ERROR ==> not all input files processed"\n' |
| 1297 |
+ |
txt += ' echo " ==> list of processed files from crab_fjr.xml differs from list in pset.cfg"\n' |
| 1298 |
+ |
txt += ' echo " ==> diff input-files.txt processed-files.txt"\n' |
| 1299 |
+ |
txt += ' fi\n' |
| 1300 |
+ |
txt += ' elif [ $executable_exit_status -ne 0 ] || [ $executable_exit_status -ne 50015 ] || [ $executable_exit_status -ne 50017 ];then\n' |
| 1301 |
+ |
txt += ' echo ">>> Executable failed $executable_exit_status"\n' |
| 1302 |
+ |
txt += ' func_exit\n' |
| 1303 |
+ |
txt += ' fi\n' |
| 1304 |
+ |
txt += '\n' |
| 1305 |
+ |
txt += 'else\n' |
| 1306 |
+ |
txt += ' echo "CRAB FrameworkJobReport crab_fjr.xml is not available, using exit code of executable from command line."\n' |
| 1307 |
|
txt += 'fi\n' |
| 1308 |
|
txt += '\n' |
| 1309 |
+ |
txt += 'echo "ExeExitCode=$executable_exit_status" | tee -a $RUNTIME_AREA/$repo\n' |
| 1310 |
+ |
txt += 'echo "EXECUTABLE_EXIT_STATUS = $executable_exit_status"\n' |
| 1311 |
+ |
txt += 'job_exit_code=$executable_exit_status\n' |
| 1312 |
+ |
|
| 1313 |
|
return txt |
| 1314 |
|
|
| 1315 |
|
def setParam_(self, param, value): |
| 1334 |
|
txt = '' |
| 1335 |
|
txt += 'echo ">>> list of expected files on output sandbox"\n' |
| 1336 |
|
listOutFiles = [] |
| 1337 |
< |
stdout = 'CMSSW_$NJob.stdout' |
| 1337 |
> |
stdout = 'CMSSW_$NJob.stdout' |
| 1338 |
|
stderr = 'CMSSW_$NJob.stderr' |
| 1339 |
|
if (self.return_data == 1): |
| 1340 |
|
for file in (self.output_file+self.output_file_sandbox): |
| 1341 |
< |
listOutFiles.append(self.numberFile_(file, '$NJob')) |
| 1341 |
> |
listOutFiles.append(numberFile(file, '$NJob')) |
| 1342 |
|
listOutFiles.append(stdout) |
| 1343 |
|
listOutFiles.append(stderr) |
| 1344 |
|
else: |
| 1345 |
|
for file in (self.output_file_sandbox): |
| 1346 |
< |
listOutFiles.append(self.numberFile_(file, '$NJob')) |
| 1346 |
> |
listOutFiles.append(numberFile(file, '$NJob')) |
| 1347 |
|
listOutFiles.append(stdout) |
| 1348 |
|
listOutFiles.append(stderr) |
| 1349 |
|
txt += 'echo "output files: '+string.join(listOutFiles,' ')+'"\n' |
| 1350 |
|
txt += 'filesToCheck="'+string.join(listOutFiles,' ')+'"\n' |
| 1351 |
|
txt += 'export filesToCheck\n' |
| 1352 |
< |
return txt |
| 1352 |
> |
return txt |
