| 5 |
|
from BlackWhiteListParser import BlackWhiteListParser |
| 6 |
|
import common |
| 7 |
|
import Scram |
| 8 |
+ |
from LFNBaseName import * |
| 9 |
|
|
| 10 |
|
import os, string, glob |
| 11 |
|
|
| 34 |
|
self.executable = '' |
| 35 |
|
self.executable_arch = self.scram.getArch() |
| 36 |
|
self.tgz_name = 'default.tgz' |
| 36 |
– |
self.additional_tgz_name = 'additional.tgz' |
| 37 |
|
self.scriptName = 'CMSSW.sh' |
| 38 |
|
self.pset = '' #scrip use case Da |
| 39 |
|
self.datasetPath = '' #scrip use case Da |
| 42 |
|
self.fjrFileName = 'crab_fjr.xml' |
| 43 |
|
|
| 44 |
|
self.version = self.scram.getSWVersion() |
| 45 |
+ |
version_array = self.version.split('_') |
| 46 |
+ |
self.major_version = 0 |
| 47 |
+ |
self.minor_version = 0 |
| 48 |
+ |
try: |
| 49 |
+ |
self.major_version = int(version_array[1]) |
| 50 |
+ |
self.minor_version = int(version_array[2]) |
| 51 |
+ |
except: |
| 52 |
+ |
msg = "Cannot parse CMSSW version string: " + "_".join(version_array) + " for major and minor release number!" |
| 53 |
+ |
raise CrabException(msg) |
| 54 |
+ |
|
| 55 |
|
|
| 56 |
|
# |
| 57 |
|
# Try to block creation in case of arch/version mismatch |
| 58 |
|
# |
| 59 |
|
|
| 60 |
< |
a = string.split(self.version, "_") |
| 61 |
< |
|
| 62 |
< |
if int(a[1]) == 1 and (int(a[2]) < 5 and self.executable_arch.find('slc4') == 0): |
| 63 |
< |
msg = "Warning: You are using %s version of CMSSW with %s architecture. \n--> Did you compile your libraries with SLC3? Otherwise you can find some problems running on SLC4 Grid nodes.\n"%(self.version, self.executable_arch) |
| 64 |
< |
common.logger.message(msg) |
| 65 |
< |
if int(a[1]) == 1 and (int(a[2]) >= 5 and self.executable_arch.find('slc3') == 0): |
| 66 |
< |
msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch) |
| 67 |
< |
raise CrabException(msg) |
| 60 |
> |
# a = string.split(self.version, "_") |
| 61 |
> |
# |
| 62 |
> |
# if int(a[1]) == 1 and (int(a[2]) < 5 and self.executable_arch.find('slc4') == 0): |
| 63 |
> |
# msg = "Warning: You are using %s version of CMSSW with %s architecture. \n--> Did you compile your libraries with SLC3? Otherwise you can find some problems running on SLC4 Grid nodes.\n"%(self.version, self.executable_arch) |
| 64 |
> |
# common.logger.message(msg) |
| 65 |
> |
# if int(a[1]) == 1 and (int(a[2]) >= 5 and self.executable_arch.find('slc3') == 0): |
| 66 |
> |
# msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch) |
| 67 |
> |
# raise CrabException(msg) |
| 68 |
> |
# |
| 69 |
|
|
| 59 |
– |
common.taskDB.setDict('codeVersion',self.version) |
| 60 |
– |
self.setParam_('application', self.version) |
| 70 |
|
|
| 71 |
|
### collect Data cards |
| 72 |
|
|
| 82 |
|
self.datasetPath = tmp |
| 83 |
|
self.selectNoInput = 0 |
| 84 |
|
|
| 76 |
– |
# ML monitoring |
| 77 |
– |
# split dataset path style: /PreProdR3Minbias/SIM/GEN-SIM |
| 78 |
– |
if not self.datasetPath: |
| 79 |
– |
self.setParam_('dataset', 'None') |
| 80 |
– |
self.setParam_('owner', 'None') |
| 81 |
– |
else: |
| 82 |
– |
## SL what is supposed to fail here? |
| 83 |
– |
try: |
| 84 |
– |
datasetpath_split = self.datasetPath.split("/") |
| 85 |
– |
# standard style |
| 86 |
– |
self.setParam_('datasetFull', self.datasetPath) |
| 87 |
– |
self.setParam_('dataset', datasetpath_split[1]) |
| 88 |
– |
self.setParam_('owner', datasetpath_split[2]) |
| 89 |
– |
except: |
| 90 |
– |
self.setParam_('dataset', self.datasetPath) |
| 91 |
– |
self.setParam_('owner', self.datasetPath) |
| 92 |
– |
|
| 93 |
– |
self.setParam_('taskId', common.taskDB.dict('taskId')) |
| 94 |
– |
|
| 85 |
|
self.dataTiers = [] |
| 86 |
|
|
| 87 |
|
## now the application |
| 88 |
|
self.executable = cfg_params.get('CMSSW.executable','cmsRun') |
| 99 |
– |
self.setParam_('exe', self.executable) |
| 89 |
|
log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable) |
| 90 |
|
|
| 91 |
|
if not cfg_params.has_key('CMSSW.pset'): |
| 122 |
|
# script_exe file as additional file in inputSandbox |
| 123 |
|
self.scriptExe = cfg_params.get('USER.script_exe',None) |
| 124 |
|
if self.scriptExe : |
| 125 |
< |
if not os.path.isfile(self.scriptExe): |
| 126 |
< |
msg ="ERROR. file "+self.scriptExe+" not found" |
| 127 |
< |
raise CrabException(msg) |
| 128 |
< |
self.additional_inbox_files.append(string.strip(self.scriptExe)) |
| 125 |
> |
if not os.path.isfile(self.scriptExe): |
| 126 |
> |
msg ="ERROR. file "+self.scriptExe+" not found" |
| 127 |
> |
raise CrabException(msg) |
| 128 |
> |
self.additional_inbox_files.append(string.strip(self.scriptExe)) |
| 129 |
|
|
| 130 |
|
#CarlosDaniele |
| 131 |
|
if self.datasetPath == None and self.pset == None and self.scriptExe == '' : |
| 132 |
< |
msg ="Error. script_exe not defined" |
| 133 |
< |
raise CrabException(msg) |
| 132 |
> |
msg ="Error. script_exe not defined" |
| 133 |
> |
raise CrabException(msg) |
| 134 |
|
|
| 135 |
|
## additional input files |
| 136 |
|
if cfg_params.has_key('USER.additional_input_files'): |
| 191 |
|
msg = 'Must specify number_of_jobs.' |
| 192 |
|
raise CrabException(msg) |
| 193 |
|
|
| 194 |
< |
## source seed for pythia |
| 194 |
> |
## New method of dealing with seeds |
| 195 |
> |
self.incrementSeeds = [] |
| 196 |
> |
self.preserveSeeds = [] |
| 197 |
> |
if cfg_params.has_key('CMSSW.preserve_seeds'): |
| 198 |
> |
tmpList = cfg_params['CMSSW.preserve_seeds'].split(',') |
| 199 |
> |
for tmp in tmpList: |
| 200 |
> |
tmp.strip() |
| 201 |
> |
self.preserveSeeds.append(tmp) |
| 202 |
> |
if cfg_params.has_key('CMSSW.increment_seeds'): |
| 203 |
> |
tmpList = cfg_params['CMSSW.increment_seeds'].split(',') |
| 204 |
> |
for tmp in tmpList: |
| 205 |
> |
tmp.strip() |
| 206 |
> |
self.incrementSeeds.append(tmp) |
| 207 |
> |
|
| 208 |
> |
## Old method of dealing with seeds |
| 209 |
> |
## FUTURE: This is for old CMSSW and old CRAB. Can throw exceptions after a couple of CRAB releases and then |
| 210 |
> |
## remove |
| 211 |
|
self.sourceSeed = cfg_params.get('CMSSW.pythia_seed',None) |
| 212 |
+ |
if self.sourceSeed: |
| 213 |
+ |
print "pythia_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds." |
| 214 |
+ |
self.incrementSeeds.append('sourceSeed') |
| 215 |
|
|
| 216 |
|
self.sourceSeedVtx = cfg_params.get('CMSSW.vtx_seed',None) |
| 217 |
+ |
if self.sourceSeedVtx: |
| 218 |
+ |
print "vtx_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds." |
| 219 |
+ |
self.incrementSeeds.append('VtxSmeared') |
| 220 |
|
|
| 221 |
|
self.sourceSeedG4 = cfg_params.get('CMSSW.g4_seed',None) |
| 222 |
+ |
if self.sourceSeedG4: |
| 223 |
+ |
print "g4_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds." |
| 224 |
+ |
self.incrementSeeds.append('g4SimHits') |
| 225 |
|
|
| 226 |
|
self.sourceSeedMix = cfg_params.get('CMSSW.mix_seed',None) |
| 227 |
+ |
if self.sourceSeedMix: |
| 228 |
+ |
print "mix_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds." |
| 229 |
+ |
self.incrementSeeds.append('mix') |
| 230 |
|
|
| 231 |
|
self.firstRun = cfg_params.get('CMSSW.first_run',None) |
| 232 |
|
|
| 251 |
|
blockSites = self.DataDiscoveryAndLocation(cfg_params) |
| 252 |
|
#DBSDLS-end |
| 253 |
|
|
| 237 |
– |
self.tgzNameWithPath = self.getTarBall(self.executable) |
| 254 |
|
|
| 255 |
|
## Select Splitting |
| 256 |
|
if self.selectNoInput: |
| 264 |
|
# modify Pset |
| 265 |
|
if self.pset != None: #CarlosDaniele |
| 266 |
|
try: |
| 267 |
< |
if (self.datasetPath): # standard job |
| 268 |
< |
# allow to processa a fraction of events in a file |
| 269 |
< |
PsetEdit.inputModule("INPUTFILE") |
| 270 |
< |
PsetEdit.maxEvent(0) |
| 255 |
< |
PsetEdit.skipEvent(0) |
| 256 |
< |
else: # pythia like job |
| 257 |
< |
PsetEdit.maxEvent(self.eventsPerJob) |
| 258 |
< |
if (self.firstRun): |
| 259 |
< |
PsetEdit.pythiaFirstRun(0) #First Run |
| 260 |
< |
if (self.sourceSeed) : |
| 261 |
< |
PsetEdit.pythiaSeed(0) |
| 262 |
< |
if (self.sourceSeedVtx) : |
| 263 |
< |
PsetEdit.vtxSeed(0) |
| 264 |
< |
if (self.sourceSeedG4) : |
| 265 |
< |
PsetEdit.g4Seed(0) |
| 266 |
< |
if (self.sourceSeedMix) : |
| 267 |
< |
PsetEdit.mixSeed(0) |
| 268 |
< |
# add FrameworkJobReport to parameter-set |
| 269 |
< |
PsetEdit.addCrabFJR(self.fjrFileName) |
| 267 |
> |
# Add FrameworkJobReport to parameter-set, set max events. |
| 268 |
> |
# Reset later for data jobs by writeCFG which does all modifications |
| 269 |
> |
PsetEdit.addCrabFJR(self.fjrFileName) # FUTURE: Job report addition not needed by CMSSW>1.5 |
| 270 |
> |
PsetEdit.maxEvent(self.eventsPerJob) |
| 271 |
|
PsetEdit.psetWriter(self.configFilename()) |
| 272 |
|
except: |
| 273 |
|
msg='Error while manipuliating ParameterSet: exiting...' |
| 274 |
|
raise CrabException(msg) |
| 275 |
+ |
self.tgzNameWithPath = self.getTarBall(self.executable) |
| 276 |
|
|
| 277 |
|
def DataDiscoveryAndLocation(self, cfg_params): |
| 278 |
|
|
| 327 |
|
|
| 328 |
|
return sites |
| 329 |
|
|
| 330 |
< |
def setArgsList(self, argsList): |
| 331 |
< |
self.argsList = argsList |
| 330 |
> |
# to Be Removed DS -- BL |
| 331 |
> |
# def setArgsList(self, argsList): |
| 332 |
> |
# self.argsList = argsList |
| 333 |
|
|
| 334 |
|
def jobSplittingByBlocks(self, blockSites): |
| 335 |
|
""" |
| 381 |
|
else : |
| 382 |
|
totalNumberOfJobs = self.ncjobs |
| 383 |
|
|
| 381 |
– |
|
| 384 |
|
blocks = blockSites.keys() |
| 385 |
|
blockCount = 0 |
| 386 |
|
# Backup variable in case self.maxEvents counted events in a non-included block |
| 435 |
|
except KeyError: |
| 436 |
|
common.logger.message("File "+str(file)+" has unknown number of events: skipping") |
| 437 |
|
|
| 438 |
< |
|
| 438 |
> |
eventsPerJobRequested = min(eventsPerJobRequested, eventsRemaining) |
| 439 |
|
# if less events in file remain than eventsPerJobRequested |
| 440 |
< |
if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested ) : |
| 440 |
> |
if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested): |
| 441 |
|
# if last file in block |
| 442 |
|
if ( fileCount == numFilesInBlock-1 ) : |
| 443 |
|
# end job using last file, use remaining events in block |
| 501 |
|
jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file]) |
| 502 |
|
# remove all but the last file |
| 503 |
|
filesEventCount = self.eventsbyfile[file] |
| 504 |
< |
parString = "" |
| 503 |
< |
parString += '\\\"' + file + '\\\"\,' |
| 504 |
> |
parString = '\\\"' + file + '\\\"\,' |
| 505 |
|
pass # END if |
| 506 |
|
pass # END while (iterate over files in the block) |
| 507 |
|
pass # END while (iterate over blocks in the dataset) |
| 521 |
|
for block in blocks: |
| 522 |
|
if block in jobsOfBlock.keys() : |
| 523 |
|
blockCounter += 1 |
| 524 |
< |
screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),','.join(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block))) |
| 524 |
> |
screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]), |
| 525 |
> |
','.join(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block))) |
| 526 |
|
if len(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)) == 0: |
| 527 |
|
noSiteBlock.append( spanRanges(jobsOfBlock[block]) ) |
| 528 |
|
bloskNoSite.append( blockCounter ) |
| 604 |
|
if (self.firstRun): |
| 605 |
|
## pythia first run |
| 606 |
|
args.append(str(self.firstRun)+str(i)) |
| 605 |
– |
if (self.sourceSeed): |
| 606 |
– |
args.append(str(self.sourceSeed)+str(i)) |
| 607 |
– |
if (self.sourceSeedVtx): |
| 608 |
– |
## + vtx random seed |
| 609 |
– |
args.append(str(self.sourceSeedVtx)+str(i)) |
| 610 |
– |
if (self.sourceSeedG4): |
| 611 |
– |
## + G4 random seed |
| 612 |
– |
args.append(str(self.sourceSeedG4)+str(i)) |
| 613 |
– |
if (self.sourceSeedMix): |
| 614 |
– |
## + Mix random seed |
| 615 |
– |
args.append(str(self.sourceSeedMix)+str(i)) |
| 616 |
– |
pass |
| 617 |
– |
pass |
| 607 |
|
self.list_of_args.append(args) |
| 619 |
– |
pass |
| 608 |
|
|
| 609 |
|
return |
| 610 |
|
|
| 634 |
|
|
| 635 |
|
def split(self, jobParams): |
| 636 |
|
|
| 649 |
– |
common.jobDB.load() |
| 637 |
|
#### Fabio |
| 638 |
|
njobs = self.total_number_of_jobs |
| 639 |
|
arglist = self.list_of_args |
| 641 |
|
for i in range(njobs): |
| 642 |
|
jobParams.append("") |
| 643 |
|
|
| 644 |
+ |
listID=[] |
| 645 |
+ |
listField=[] |
| 646 |
|
for job in range(njobs): |
| 647 |
|
jobParams[job] = arglist[job] |
| 648 |
< |
# print str(arglist[job]) |
| 649 |
< |
# print jobParams[job] |
| 650 |
< |
common.jobDB.setArguments(job, jobParams[job]) |
| 651 |
< |
common.logger.debug(5,"Job "+str(job)+" Destination: "+str(self.jobDestination[job])) |
| 652 |
< |
common.jobDB.setDestination(job, self.jobDestination[job]) |
| 648 |
> |
listID.append(job+1) |
| 649 |
> |
job_ToSave ={} |
| 650 |
> |
concString = ' ' |
| 651 |
> |
argu='' |
| 652 |
> |
if len(jobParams[job]): |
| 653 |
> |
argu += concString.join(jobParams[job] ) |
| 654 |
> |
job_ToSave['arguments']= str(job+1)+' '+argu## new BL--DS |
| 655 |
> |
job_ToSave['dlsDestination']= self.jobDestination[job]## new BL--DS |
| 656 |
> |
#common._db.updateJob_(job,job_ToSave)## new BL--DS |
| 657 |
> |
listField.append(job_ToSave) |
| 658 |
> |
msg="Job "+str(job)+" Arguments: "+str(job+1)+" "+argu+"\n" \ |
| 659 |
> |
+" Destination: "+str(self.jobDestination[job]) |
| 660 |
> |
common.logger.debug(5,msg) |
| 661 |
> |
#common.logger.debug(5,"Job "+str(job)+" Destination: "+str(self.jobDestination[job])) |
| 662 |
> |
common._db.updateJob_(listID,listField)## new BL--DS |
| 663 |
> |
## Pay Attention Here....DS--BL |
| 664 |
> |
self.argsList = (len(jobParams[0])+1) |
| 665 |
|
|
| 665 |
– |
common.jobDB.save() |
| 666 |
|
return |
| 667 |
|
|
| 668 |
– |
def getJobTypeArguments(self, nj, sched): |
| 669 |
– |
result = '' |
| 670 |
– |
for i in common.jobDB.arguments(nj): |
| 671 |
– |
result=result+str(i)+" " |
| 672 |
– |
return result |
| 673 |
– |
|
| 668 |
|
def numberOfJobs(self): |
| 669 |
|
# Fabio |
| 670 |
|
return self.total_number_of_jobs |
| 699 |
|
|
| 700 |
|
## check if working area is release top |
| 701 |
|
if swReleaseTop == '' or swArea == swReleaseTop: |
| 702 |
+ |
common.logger.debug(3,"swArea = "+swArea+" swReleaseTop ="+swReleaseTop) |
| 703 |
|
return |
| 704 |
|
|
| 705 |
|
import tarfile |
| 742 |
|
|
| 743 |
|
## Now check if any data dir(s) is present |
| 744 |
|
swAreaLen=len(swArea) |
| 745 |
+ |
self.dataExist = False |
| 746 |
|
for root, dirs, files in os.walk(swArea): |
| 747 |
|
if "data" in dirs: |
| 748 |
+ |
self.dataExist=True |
| 749 |
|
common.logger.debug(5,"data "+root+"/data"+" to be tarred") |
| 750 |
|
tar.add(root+"/data",root[swAreaLen:]+"/data") |
| 751 |
|
|
| 752 |
< |
### Removed ProdAgent Api dependencies ### |
| 753 |
< |
### Add ProdAgent dir to tar |
| 754 |
< |
#paDir = 'ProdAgentApi' |
| 755 |
< |
#pa = os.environ['CRABDIR'] + '/' + 'ProdAgentApi' |
| 756 |
< |
#if os.path.isdir(pa): |
| 757 |
< |
# tar.add(pa,paDir) |
| 752 |
> |
### CMSSW ParameterSet |
| 753 |
> |
if not self.pset is None: |
| 754 |
> |
cfg_file = common.work_space.jobDir()+self.configFilename() |
| 755 |
> |
tar.add(cfg_file,self.configFilename()) |
| 756 |
> |
common.logger.debug(5,"File added to "+self.tgzNameWithPath+" : "+str(tar.getnames())) |
| 757 |
> |
|
| 758 |
|
|
| 759 |
|
## Add ProdCommon dir to tar |
| 760 |
|
prodcommonDir = 'ProdCommon' |
| 761 |
|
prodcommonPath = os.environ['CRABDIR'] + '/' + 'ProdCommon' |
| 762 |
|
if os.path.isdir(prodcommonPath): |
| 763 |
|
tar.add(prodcommonPath,prodcommonDir) |
| 764 |
+ |
common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames())) |
| 765 |
+ |
|
| 766 |
+ |
##### ML stuff |
| 767 |
+ |
ML_file_list=['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py'] |
| 768 |
+ |
path=os.environ['CRABDIR'] + '/python/' |
| 769 |
+ |
for file in ML_file_list: |
| 770 |
+ |
tar.add(path+file,file) |
| 771 |
+ |
common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames())) |
| 772 |
+ |
|
| 773 |
+ |
##### Utils |
| 774 |
+ |
Utils_file_list=['parseCrabFjr.py','writeCfg.py', 'JobReportErrorCode.py'] |
| 775 |
+ |
for file in Utils_file_list: |
| 776 |
+ |
tar.add(path+file,file) |
| 777 |
+ |
common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames())) |
| 778 |
|
|
| 779 |
+ |
##### AdditionalFiles |
| 780 |
+ |
for file in self.additional_inbox_files: |
| 781 |
+ |
tar.add(file,string.split(file,'/')[-1]) |
| 782 |
|
common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames())) |
| 783 |
+ |
|
| 784 |
|
tar.close() |
| 785 |
|
except : |
| 786 |
|
raise CrabException('Could not create tar-ball') |
| 791 |
|
raise CrabException('Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' + str(self.MaxTarBallSize) + ' MB input sandbox limit and not supported by the used GRID submission system. Please make sure that no unnecessary files are in all data directories in your local CMSSW project area as they are automatically packed into the input sandbox.') |
| 792 |
|
|
| 793 |
|
## create tar-ball with ML stuff |
| 779 |
– |
self.MLtgzfile = common.work_space.pathForTgz()+'share/MLfiles.tgz' |
| 780 |
– |
try: |
| 781 |
– |
tar = tarfile.open(self.MLtgzfile, "w:gz") |
| 782 |
– |
path=os.environ['CRABDIR'] + '/python/' |
| 783 |
– |
for file in ['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py', 'parseCrabFjr.py']: |
| 784 |
– |
tar.add(path+file,file) |
| 785 |
– |
common.logger.debug(5,"Files added to "+self.MLtgzfile+" : "+str(tar.getnames())) |
| 786 |
– |
tar.close() |
| 787 |
– |
except : |
| 788 |
– |
raise CrabException('Could not create ML files tar-ball') |
| 789 |
– |
|
| 790 |
– |
return |
| 791 |
– |
|
| 792 |
– |
def additionalInputFileTgz(self): |
| 793 |
– |
""" |
| 794 |
– |
Put all additional files into a tar ball and return its name |
| 795 |
– |
""" |
| 796 |
– |
import tarfile |
| 797 |
– |
tarName= common.work_space.pathForTgz()+'share/'+self.additional_tgz_name |
| 798 |
– |
tar = tarfile.open(tarName, "w:gz") |
| 799 |
– |
for file in self.additional_inbox_files: |
| 800 |
– |
tar.add(file,string.split(file,'/')[-1]) |
| 801 |
– |
common.logger.debug(5,"Files added to "+self.additional_tgz_name+" : "+str(tar.getnames())) |
| 802 |
– |
tar.close() |
| 803 |
– |
return tarName |
| 794 |
|
|
| 795 |
< |
def wsSetupEnvironment(self, nj): |
| 795 |
> |
def wsSetupEnvironment(self, nj=0): |
| 796 |
|
""" |
| 797 |
|
Returns part of a job script which prepares |
| 798 |
|
the execution environment for the job 'nj'. |
| 799 |
|
""" |
| 800 |
|
# Prepare JobType-independent part |
| 801 |
< |
txt = '' |
| 801 |
> |
txt = '\n#Written by cms_cmssw::wsSetupEnvironment\n' |
| 802 |
|
txt += 'echo ">>> setup environment"\n' |
| 803 |
|
txt += 'if [ $middleware == LCG ]; then \n' |
| 804 |
|
txt += self.wsSetupCMSLCGEnvironment_() |
| 805 |
|
txt += 'elif [ $middleware == OSG ]; then\n' |
| 806 |
|
txt += ' WORKING_DIR=`/bin/mktemp -d $OSG_WN_TMP/cms_XXXXXXXXXXXX`\n' |
| 807 |
|
txt += ' if [ ! $? == 0 ] ;then\n' |
| 808 |
< |
txt += ' echo "SET_CMS_ENV 10016 ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n' |
| 809 |
< |
txt += ' echo "JOB_EXIT_STATUS = 10016"\n' |
| 810 |
< |
txt += ' echo "JobExitCode=10016" | tee -a $RUNTIME_AREA/$repo\n' |
| 821 |
< |
txt += ' dumpStatus $RUNTIME_AREA/$repo\n' |
| 822 |
< |
txt += ' exit 1\n' |
| 808 |
> |
txt += ' echo "ERROR ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n' |
| 809 |
> |
txt += ' job_exit_code=10016\n' |
| 810 |
> |
txt += ' func_exit\n' |
| 811 |
|
txt += ' fi\n' |
| 812 |
|
txt += ' echo ">>> Created working directory: $WORKING_DIR"\n' |
| 813 |
|
txt += '\n' |
| 827 |
|
txt += scram+' project CMSSW '+self.version+'\n' |
| 828 |
|
txt += 'status=$?\n' |
| 829 |
|
txt += 'if [ $status != 0 ] ; then\n' |
| 830 |
< |
txt += ' echo "SET_EXE_ENV 10034 ==>ERROR CMSSW '+self.version+' not found on `hostname`" \n' |
| 831 |
< |
txt += ' echo "JOB_EXIT_STATUS = 10034"\n' |
| 832 |
< |
txt += ' echo "JobExitCode=10034" | tee -a $RUNTIME_AREA/$repo\n' |
| 845 |
< |
txt += ' dumpStatus $RUNTIME_AREA/$repo\n' |
| 846 |
< |
txt += ' if [ $middleware == OSG ]; then \n' |
| 847 |
< |
txt += ' cd $RUNTIME_AREA\n' |
| 848 |
< |
txt += ' echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n' |
| 849 |
< |
txt += ' echo ">>> Remove working directory: $WORKING_DIR"\n' |
| 850 |
< |
txt += ' /bin/rm -rf $WORKING_DIR\n' |
| 851 |
< |
txt += ' if [ -d $WORKING_DIR ] ;then\n' |
| 852 |
< |
txt += ' echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n' |
| 853 |
< |
txt += ' echo "JOB_EXIT_STATUS = 10018"\n' |
| 854 |
< |
txt += ' echo "JobExitCode=10018" | tee -a $RUNTIME_AREA/$repo\n' |
| 855 |
< |
txt += ' dumpStatus $RUNTIME_AREA/$repo\n' |
| 856 |
< |
txt += ' fi\n' |
| 857 |
< |
txt += ' fi \n' |
| 858 |
< |
txt += ' exit 1 \n' |
| 830 |
> |
txt += ' echo "ERROR ==> CMSSW '+self.version+' not found on `hostname`" \n' |
| 831 |
> |
txt += ' job_exit_code=10034\n' |
| 832 |
> |
txt += ' func_exit\n' |
| 833 |
|
txt += 'fi \n' |
| 834 |
|
txt += 'cd '+self.version+'\n' |
| 861 |
– |
########## FEDE FOR DBS2 ###################### |
| 835 |
|
txt += 'SOFTWARE_DIR=`pwd`\n' |
| 836 |
|
txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n' |
| 864 |
– |
############################################### |
| 865 |
– |
### needed grep for bug in scramv1 ### |
| 837 |
|
txt += 'eval `'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME`\n' |
| 838 |
+ |
txt += 'if [ $? != 0 ] ; then\n' |
| 839 |
+ |
txt += ' echo "ERROR ==> Problem with the command: "\n' |
| 840 |
+ |
txt += ' echo "eval \`'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME \` at `hostname`"\n' |
| 841 |
+ |
txt += ' job_exit_code=10034\n' |
| 842 |
+ |
txt += ' func_exit\n' |
| 843 |
+ |
txt += 'fi \n' |
| 844 |
|
# Handle the arguments: |
| 845 |
|
txt += "\n" |
| 846 |
|
txt += "## number of arguments (first argument always jobnumber)\n" |
| 847 |
|
txt += "\n" |
| 848 |
< |
txt += "if [ $nargs -lt "+str(len(self.argsList[nj].split()))+" ]\n" |
| 848 |
> |
txt += "if [ $nargs -lt "+str(self.argsList)+" ]\n" |
| 849 |
|
txt += "then\n" |
| 850 |
< |
txt += " echo 'SET_EXE_ENV 1 ==> ERROR Too few arguments' +$nargs+ \n" |
| 851 |
< |
txt += ' echo "JOB_EXIT_STATUS = 50113"\n' |
| 852 |
< |
txt += ' echo "JobExitCode=50113" | tee -a $RUNTIME_AREA/$repo\n' |
| 876 |
< |
txt += ' dumpStatus $RUNTIME_AREA/$repo\n' |
| 877 |
< |
txt += ' if [ $middleware == OSG ]; then \n' |
| 878 |
< |
txt += ' cd $RUNTIME_AREA\n' |
| 879 |
< |
txt += ' echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n' |
| 880 |
< |
txt += ' echo ">>> Remove working directory: $WORKING_DIR"\n' |
| 881 |
< |
txt += ' /bin/rm -rf $WORKING_DIR\n' |
| 882 |
< |
txt += ' if [ -d $WORKING_DIR ] ;then\n' |
| 883 |
< |
txt += ' echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n' |
| 884 |
< |
txt += ' echo "JOB_EXIT_STATUS = 50114"\n' |
| 885 |
< |
txt += ' echo "JobExitCode=50114" | tee -a $RUNTIME_AREA/$repo\n' |
| 886 |
< |
txt += ' dumpStatus $RUNTIME_AREA/$repo\n' |
| 887 |
< |
txt += ' fi\n' |
| 888 |
< |
txt += ' fi \n' |
| 889 |
< |
txt += " exit 1\n" |
| 850 |
> |
txt += " echo 'ERROR ==> Too few arguments' +$nargs+ \n" |
| 851 |
> |
txt += ' job_exit_code=50113\n' |
| 852 |
> |
txt += " func_exit\n" |
| 853 |
|
txt += "fi\n" |
| 854 |
|
txt += "\n" |
| 855 |
|
|
| 871 |
|
txt += 'PrimaryDataset=null\n' |
| 872 |
|
txt += 'DataTier=null\n' |
| 873 |
|
txt += 'ApplicationFamily=MCDataTier\n' |
| 874 |
< |
if self.pset != None: #CarlosDaniele |
| 874 |
> |
if self.pset != None: |
| 875 |
|
pset = os.path.basename(job.configFilename()) |
| 876 |
|
txt += '\n' |
| 877 |
|
txt += 'cp $RUNTIME_AREA/'+pset+' .\n' |
| 878 |
|
if (self.datasetPath): # standard job |
| 879 |
< |
txt += 'InputFiles=${args[1]}\n' |
| 880 |
< |
txt += 'MaxEvents=${args[2]}\n' |
| 881 |
< |
txt += 'SkipEvents=${args[3]}\n' |
| 879 |
> |
txt += 'InputFiles=${args[1]}; export InputFiles\n' |
| 880 |
> |
txt += 'MaxEvents=${args[2]}; export MaxEvents\n' |
| 881 |
> |
txt += 'SkipEvents=${args[3]}; export SkipEvents\n' |
| 882 |
|
txt += 'echo "Inputfiles:<$InputFiles>"\n' |
| 920 |
– |
txt += 'sed "s#\'INPUTFILE\'#$InputFiles#" '+pset+' > tmp && mv -f tmp '+pset+'\n' |
| 883 |
|
txt += 'echo "MaxEvents:<$MaxEvents>"\n' |
| 922 |
– |
txt += 'sed "s#int32 input = 0#int32 input = $MaxEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n' |
| 884 |
|
txt += 'echo "SkipEvents:<$SkipEvents>"\n' |
| 924 |
– |
txt += 'sed "s#uint32 skipEvents = 0#uint32 skipEvents = $SkipEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n' |
| 885 |
|
else: # pythia like job |
| 886 |
< |
seedIndex=1 |
| 886 |
> |
txt += 'PreserveSeeds=' + ','.join(self.preserveSeeds) + '; export PreserveSeeds\n' |
| 887 |
> |
txt += 'IncrementSeeds=' + ','.join(self.incrementSeeds) + '; export IncrementSeeds\n' |
| 888 |
> |
txt += 'echo "PreserveSeeds: <$PreserveSeeds>"\n' |
| 889 |
> |
txt += 'echo "IncrementSeeds:<$IncrementSeeds>"\n' |
| 890 |
|
if (self.firstRun): |
| 891 |
< |
txt += 'FirstRun=${args['+str(seedIndex)+']}\n' |
| 891 |
> |
txt += 'FirstRun=${args[1]}; export FirstRun\n' |
| 892 |
|
txt += 'echo "FirstRun: <$FirstRun>"\n' |
| 930 |
– |
txt += 'sed "s#uint32 firstRun = 0#uint32 firstRun = $FirstRun#" '+pset+' > tmp && mv -f tmp '+pset+'\n' |
| 931 |
– |
seedIndex=seedIndex+1 |
| 893 |
|
|
| 933 |
– |
if (self.sourceSeed): |
| 934 |
– |
txt += 'Seed=${args['+str(seedIndex)+']}\n' |
| 935 |
– |
txt += 'sed "s#uint32 sourceSeed = 0#uint32 sourceSeed = $Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n' |
| 936 |
– |
seedIndex=seedIndex+1 |
| 937 |
– |
## the following seeds are not always present |
| 938 |
– |
if (self.sourceSeedVtx): |
| 939 |
– |
txt += 'VtxSeed=${args['+str(seedIndex)+']}\n' |
| 940 |
– |
txt += 'echo "VtxSeed: <$VtxSeed>"\n' |
| 941 |
– |
txt += 'sed "s#uint32 VtxSmeared = 0#uint32 VtxSmeared = $VtxSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n' |
| 942 |
– |
seedIndex += 1 |
| 943 |
– |
if (self.sourceSeedG4): |
| 944 |
– |
txt += 'G4Seed=${args['+str(seedIndex)+']}\n' |
| 945 |
– |
txt += 'echo "G4Seed: <$G4Seed>"\n' |
| 946 |
– |
txt += 'sed "s#uint32 g4SimHits = 0#uint32 g4SimHits = $G4Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n' |
| 947 |
– |
seedIndex += 1 |
| 948 |
– |
if (self.sourceSeedMix): |
| 949 |
– |
txt += 'mixSeed=${args['+str(seedIndex)+']}\n' |
| 950 |
– |
txt += 'echo "MixSeed: <$mixSeed>"\n' |
| 951 |
– |
txt += 'sed "s#uint32 mix = 0#uint32 mix = $mixSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n' |
| 952 |
– |
seedIndex += 1 |
| 953 |
– |
pass |
| 954 |
– |
pass |
| 894 |
|
txt += 'mv -f '+pset+' pset.cfg\n' |
| 895 |
|
|
| 957 |
– |
if len(self.additional_inbox_files) > 0: |
| 958 |
– |
txt += 'if [ -e $RUNTIME_AREA/'+self.additional_tgz_name+' ] ; then\n' |
| 959 |
– |
txt += ' tar xzvf $RUNTIME_AREA/'+self.additional_tgz_name+'\n' |
| 960 |
– |
txt += 'fi\n' |
| 961 |
– |
pass |
| 896 |
|
|
| 897 |
< |
if self.pset != None: #CarlosDaniele |
| 897 |
> |
if self.pset != None: |
| 898 |
|
txt += '\n' |
| 899 |
|
txt += 'echo "***** cat pset.cfg *********"\n' |
| 900 |
|
txt += 'cat pset.cfg\n' |
| 901 |
|
txt += 'echo "****** end pset.cfg ********"\n' |
| 902 |
|
txt += '\n' |
| 969 |
– |
### FEDE FOR DBS OUTPUT PUBLICATION |
| 903 |
|
txt += 'PSETHASH=`EdmConfigHash < pset.cfg` \n' |
| 904 |
|
txt += 'echo "PSETHASH = $PSETHASH" \n' |
| 972 |
– |
############## |
| 905 |
|
txt += '\n' |
| 906 |
|
return txt |
| 907 |
|
|
| 908 |
< |
def wsBuildExe(self, nj=0): |
| 908 |
> |
def wsUntarSoftware(self, nj=0): |
| 909 |
|
""" |
| 910 |
|
Put in the script the commands to build an executable |
| 911 |
|
or a library. |
| 912 |
|
""" |
| 913 |
|
|
| 914 |
< |
txt = "" |
| 914 |
> |
txt = '\n#Written by cms_cmssw::wsUntarSoftware\n' |
| 915 |
|
|
| 916 |
|
if os.path.isfile(self.tgzNameWithPath): |
| 917 |
|
txt += 'echo ">>> tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+' :" \n' |
| 918 |
|
txt += 'tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n' |
| 919 |
+ |
txt += 'ls -Al \n' |
| 920 |
|
txt += 'untar_status=$? \n' |
| 921 |
|
txt += 'if [ $untar_status -ne 0 ]; then \n' |
| 922 |
< |
txt += ' echo "SET_EXE 1 ==> ERROR Untarring .tgz file failed"\n' |
| 923 |
< |
txt += ' echo "JOB_EXIT_STATUS = $untar_status" \n' |
| 924 |
< |
txt += ' echo "JobExitCode=$untar_status" | tee -a $RUNTIME_AREA/$repo\n' |
| 992 |
< |
txt += ' if [ $middleware == OSG ]; then \n' |
| 993 |
< |
txt += ' cd $RUNTIME_AREA\n' |
| 994 |
< |
txt += ' echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n' |
| 995 |
< |
txt += ' echo ">>> Remove working directory: $WORKING_DIR"\n' |
| 996 |
< |
txt += ' /bin/rm -rf $WORKING_DIR\n' |
| 997 |
< |
txt += ' if [ -d $WORKING_DIR ] ;then\n' |
| 998 |
< |
txt += ' echo "SET_EXE 50999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Untarring .tgz file failed"\n' |
| 999 |
< |
txt += ' echo "JOB_EXIT_STATUS = 50999"\n' |
| 1000 |
< |
txt += ' echo "JobExitCode=50999" | tee -a $RUNTIME_AREA/$repo\n' |
| 1001 |
< |
txt += ' dumpStatus $RUNTIME_AREA/$repo\n' |
| 1002 |
< |
txt += ' fi\n' |
| 1003 |
< |
txt += ' fi \n' |
| 1004 |
< |
txt += ' \n' |
| 1005 |
< |
txt += ' exit 1 \n' |
| 922 |
> |
txt += ' echo "ERROR ==> Untarring .tgz file failed"\n' |
| 923 |
> |
txt += ' job_exit_code=$untar_status\n' |
| 924 |
> |
txt += ' func_exit\n' |
| 925 |
|
txt += 'else \n' |
| 926 |
|
txt += ' echo "Successful untar" \n' |
| 927 |
|
txt += 'fi \n' |
| 928 |
|
txt += '\n' |
| 1010 |
– |
#### Removed ProdAgent API dependencies |
| 929 |
|
txt += 'echo ">>> Include ProdCommon in PYTHONPATH:"\n' |
| 930 |
|
txt += 'if [ -z "$PYTHONPATH" ]; then\n' |
| 931 |
< |
#### FEDE FOR DBS OUTPUT PUBLICATION |
| 1014 |
< |
txt += ' export PYTHONPATH=$SOFTWARE_DIR/ProdCommon\n' |
| 931 |
> |
txt += ' export PYTHONPATH=$RUNTIME_AREA/ProdCommon\n' |
| 932 |
|
txt += 'else\n' |
| 933 |
< |
txt += ' export PYTHONPATH=$SOFTWARE_DIR/ProdCommon:${PYTHONPATH}\n' |
| 933 |
> |
txt += ' export PYTHONPATH=$RUNTIME_AREA/ProdCommon:${PYTHONPATH}\n' |
| 934 |
|
txt += 'echo "PYTHONPATH=$PYTHONPATH"\n' |
| 1018 |
– |
################### |
| 935 |
|
txt += 'fi\n' |
| 936 |
|
txt += '\n' |
| 937 |
|
|
| 939 |
|
|
| 940 |
|
return txt |
| 941 |
|
|
| 942 |
+ |
def wsBuildExe(self, nj=0): |
| 943 |
+ |
""" |
| 944 |
+ |
Put in the script the commands to build an executable |
| 945 |
+ |
or a library. |
| 946 |
+ |
""" |
| 947 |
+ |
|
| 948 |
+ |
txt = '\n#Written by cms_cmssw::wsBuildExe\n' |
| 949 |
+ |
txt += 'echo ">>> moving CMSSW software directories in `pwd`" \n' |
| 950 |
+ |
|
| 951 |
+ |
txt += 'rm -r lib/ module/ \n' |
| 952 |
+ |
txt += 'mv $RUNTIME_AREA/lib/ . \n' |
| 953 |
+ |
txt += 'mv $RUNTIME_AREA/module/ . \n' |
| 954 |
+ |
if self.dataExist == True: txt += 'mv $RUNTIME_AREA/src/ . \n' |
| 955 |
+ |
if len(self.additional_inbox_files)>0: |
| 956 |
+ |
for file in self.additional_inbox_files: |
| 957 |
+ |
txt += 'mv $RUNTIME_AREA/'+file+' . \n' |
| 958 |
+ |
txt += 'mv $RUNTIME_AREA/ProdCommon/ . \n' |
| 959 |
+ |
|
| 960 |
+ |
txt += 'if [ -z "$PYTHONPATH" ]; then\n' |
| 961 |
+ |
txt += ' export PYTHONPATH=$SOFTWARE_DIR/ProdCommon\n' |
| 962 |
+ |
txt += 'else\n' |
| 963 |
+ |
txt += ' export PYTHONPATH=$SOFTWARE_DIR/ProdCommon:${PYTHONPATH}\n' |
| 964 |
+ |
txt += 'echo "PYTHONPATH=$PYTHONPATH"\n' |
| 965 |
+ |
txt += 'fi\n' |
| 966 |
+ |
txt += '\n' |
| 967 |
+ |
|
| 968 |
+ |
return txt |
| 969 |
+ |
|
| 970 |
|
def modifySteeringCards(self, nj): |
| 971 |
|
""" |
| 972 |
|
modify the card provided by the user, |
| 980 |
|
return self.executable |
| 981 |
|
|
| 982 |
|
def executableArgs(self): |
| 983 |
+ |
# FUTURE: This function tests the CMSSW version. Can be simplified as we drop support for old versions |
| 984 |
|
if self.scriptExe:#CarlosDaniele |
| 985 |
|
return self.scriptExe + " $NJob" |
| 986 |
|
else: |
| 987 |
< |
# if >= CMSSW_1_5_X, add -j crab_fjr.xml |
| 988 |
< |
version_array = self.scram.getSWVersion().split('_') |
| 989 |
< |
major = 0 |
| 990 |
< |
minor = 0 |
| 991 |
< |
try: |
| 992 |
< |
major = int(version_array[1]) |
| 993 |
< |
minor = int(version_array[2]) |
| 994 |
< |
except: |
| 1050 |
< |
msg = "Cannot parse CMSSW version string: " + "_".join(version_array) + " for major and minor release number!" |
| 1051 |
< |
raise CrabException(msg) |
| 1052 |
< |
if major >= 1 and minor >= 5 : |
| 1053 |
< |
return " -j " + self.fjrFileName + " -p pset.cfg" |
| 987 |
> |
ex_args = "" |
| 988 |
> |
# FUTURE: This tests the CMSSW version. Can remove code as versions deprecated |
| 989 |
> |
# Framework job report |
| 990 |
> |
if (self.major_version >= 1 and self.minor_version >= 5) or (self.major_version >= 2): |
| 991 |
> |
ex_args += " -j $RUNTIME_AREA/crab_fjr_$NJob.xml" |
| 992 |
> |
# Type of cfg file |
| 993 |
> |
if self.major_version >= 2 : |
| 994 |
> |
ex_args += " -p pset.py" |
| 995 |
|
else: |
| 996 |
< |
return " -p pset.cfg" |
| 996 |
> |
ex_args += " -p pset.cfg" |
| 997 |
> |
return ex_args |
| 998 |
|
|
| 999 |
|
def inputSandbox(self, nj): |
| 1000 |
|
""" |
| 1006 |
|
## code |
| 1007 |
|
if os.path.isfile(self.tgzNameWithPath): |
| 1008 |
|
inp_box.append(self.tgzNameWithPath) |
| 1009 |
< |
if os.path.isfile(self.MLtgzfile): |
| 1010 |
< |
inp_box.append(self.MLtgzfile) |
| 1069 |
< |
## config |
| 1070 |
< |
if not self.pset is None: |
| 1071 |
< |
inp_box.append(common.work_space.pathForTgz() + 'job/' + self.configFilename()) |
| 1072 |
< |
## additional input files |
| 1073 |
< |
tgz = self.additionalInputFileTgz() |
| 1074 |
< |
inp_box.append(tgz) |
| 1009 |
> |
wrapper = os.path.basename(str(common._db.queryTask('scriptName'))) |
| 1010 |
> |
inp_box.append(common.work_space.pathForTgz() +'job/'+ wrapper) |
| 1011 |
|
return inp_box |
| 1012 |
|
|
| 1013 |
|
def outputSandbox(self, nj): |
| 1033 |
|
Returns part of a job script which renames the produced files. |
| 1034 |
|
""" |
| 1035 |
|
|
| 1036 |
< |
txt = '\n' |
| 1036 |
> |
txt = '\n#Written by cms_cmssw::wsRenameOutput\n' |
| 1037 |
|
txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n' |
| 1038 |
|
txt += 'echo ">>> current directory content:"\n' |
| 1039 |
|
txt += 'ls \n' |
| 1040 |
|
txt += '\n' |
| 1041 |
|
|
| 1106 |
– |
txt += 'output_exit_status=0\n' |
| 1107 |
– |
|
| 1108 |
– |
for fileWithSuffix in (self.output_file_sandbox): |
| 1109 |
– |
output_file_num = self.numberFile_(fileWithSuffix, '$NJob') |
| 1110 |
– |
txt += '\n' |
| 1111 |
– |
txt += '# check output file\n' |
| 1112 |
– |
txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n' |
| 1113 |
– |
txt += ' mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n' |
| 1114 |
– |
txt += ' ln -s $RUNTIME_AREA/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n' |
| 1115 |
– |
txt += 'else\n' |
| 1116 |
– |
txt += ' exit_status=60302\n' |
| 1117 |
– |
txt += ' echo "ERROR: Output file '+fileWithSuffix+' not found"\n' |
| 1118 |
– |
if common.scheduler.name() == 'CONDOR_G': |
| 1119 |
– |
txt += ' if [ $middleware == OSG ]; then \n' |
| 1120 |
– |
txt += ' echo "prepare dummy output file"\n' |
| 1121 |
– |
txt += ' echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n' |
| 1122 |
– |
txt += ' fi \n' |
| 1123 |
– |
txt += 'fi\n' |
| 1124 |
– |
|
| 1042 |
|
for fileWithSuffix in (self.output_file): |
| 1043 |
|
output_file_num = self.numberFile_(fileWithSuffix, '$NJob') |
| 1044 |
|
txt += '\n' |
| 1051 |
|
txt += ' mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n' |
| 1052 |
|
txt += ' ln -s $RUNTIME_AREA/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n' |
| 1053 |
|
txt += 'else\n' |
| 1054 |
< |
txt += ' exit_status=60302\n' |
| 1055 |
< |
txt += ' echo "ERROR: Output file '+fileWithSuffix+' not found"\n' |
| 1056 |
< |
txt += ' echo "JOB_EXIT_STATUS = $exit_status"\n' |
| 1140 |
< |
txt += ' output_exit_status=$exit_status\n' |
| 1141 |
< |
if common.scheduler.name() == 'CONDOR_G': |
| 1054 |
> |
txt += ' job_exit_code=60302\n' |
| 1055 |
> |
txt += ' echo "WARNING: Output file '+fileWithSuffix+' not found"\n' |
| 1056 |
> |
if common.scheduler.name().upper() == 'CONDOR_G': |
| 1057 |
|
txt += ' if [ $middleware == OSG ]; then \n' |
| 1058 |
|
txt += ' echo "prepare dummy output file"\n' |
| 1059 |
|
txt += ' echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n' |
| 1108 |
|
'", other.GlueHostApplicationSoftwareRunTimeEnvironment)' |
| 1109 |
|
|
| 1110 |
|
req = req + ' && (other.GlueHostNetworkAdapterOutboundIP)' |
| 1111 |
+ |
if common.scheduler.name() == "glitecoll": |
| 1112 |
+ |
req += ' && other.GlueCEStateStatus == "Production" ' |
| 1113 |
|
|
| 1114 |
|
return req |
| 1115 |
|
|
| 1116 |
|
def configFilename(self): |
| 1117 |
|
""" return the config filename """ |
| 1118 |
< |
return self.name()+'.cfg' |
| 1118 |
> |
# FUTURE: Can remove cfg mode for CMSSW >= 2_1_x |
| 1119 |
> |
if (self.major_version >= 2 and self.minor_version >= 1) or (self.major_version >= 3): |
| 1120 |
> |
return self.name()+'.py' |
| 1121 |
> |
else: |
| 1122 |
> |
return self.name()+'.cfg' |
| 1123 |
|
|
| 1124 |
|
def wsSetupCMSOSGEnvironment_(self): |
| 1125 |
|
""" |
| 1126 |
|
Returns part of a job script which is prepares |
| 1127 |
|
the execution environment and which is common for all CMS jobs. |
| 1128 |
|
""" |
| 1129 |
< |
txt = ' echo ">>> setup CMS OSG environment:"\n' |
| 1129 |
> |
txt = '\n#Written by cms_cmssw::wsSetupCMSOSGEnvironment_\n' |
| 1130 |
> |
txt += ' echo ">>> setup CMS OSG environment:"\n' |
| 1131 |
|
txt += ' echo "set SCRAM ARCH to ' + self.executable_arch + '"\n' |
| 1132 |
|
txt += ' export SCRAM_ARCH='+self.executable_arch+'\n' |
| 1133 |
|
txt += ' echo "SCRAM_ARCH = $SCRAM_ARCH"\n' |
| 1135 |
|
txt += ' # Use $OSG_APP/cmssoft/cms/cmsset_default.sh to setup cms software\n' |
| 1136 |
|
txt += ' source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n' |
| 1137 |
|
txt += ' else\n' |
| 1138 |
< |
txt += ' echo "SET_CMS_ENV 10020 ==> ERROR $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n' |
| 1139 |
< |
txt += ' echo "JOB_EXIT_STATUS = 10020"\n' |
| 1140 |
< |
txt += ' echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n' |
| 1219 |
< |
txt += ' dumpStatus $RUNTIME_AREA/$repo\n' |
| 1220 |
< |
txt += '\n' |
| 1221 |
< |
txt += ' cd $RUNTIME_AREA\n' |
| 1222 |
< |
txt += ' echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n' |
| 1223 |
< |
txt += ' echo ">>> Remove working directory: $WORKING_DIR"\n' |
| 1224 |
< |
txt += ' /bin/rm -rf $WORKING_DIR\n' |
| 1225 |
< |
txt += ' if [ -d $WORKING_DIR ] ;then\n' |
| 1226 |
< |
txt += ' echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n' |
| 1227 |
< |
txt += ' echo "JOB_EXIT_STATUS = 10017"\n' |
| 1228 |
< |
txt += ' echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n' |
| 1229 |
< |
txt += ' dumpStatus $RUNTIME_AREA/$repo\n' |
| 1230 |
< |
txt += ' fi\n' |
| 1231 |
< |
txt += '\n' |
| 1232 |
< |
txt += ' exit 1\n' |
| 1138 |
> |
txt += ' echo "ERROR ==> $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n' |
| 1139 |
> |
txt += ' job_exit_code=10020\n' |
| 1140 |
> |
txt += ' func_exit\n' |
| 1141 |
|
txt += ' fi\n' |
| 1142 |
|
txt += '\n' |
| 1143 |
< |
txt += ' echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n' |
| 1143 |
> |
txt += ' echo "==> setup cms environment ok"\n' |
| 1144 |
|
txt += ' echo "SCRAM_ARCH = $SCRAM_ARCH"\n' |
| 1145 |
|
|
| 1146 |
|
return txt |
| 1151 |
|
Returns part of a job script which is prepares |
| 1152 |
|
the execution environment and which is common for all CMS jobs. |
| 1153 |
|
""" |
| 1154 |
< |
txt = ' echo ">>> setup CMS LCG environment:"\n' |
| 1154 |
> |
txt = '\n#Written by cms_cmssw::wsSetupCMSLCGEnvironment_\n' |
| 1155 |
> |
txt += ' echo ">>> setup CMS LCG environment:"\n' |
| 1156 |
|
txt += ' echo "set SCRAM ARCH and BUILD_ARCH to ' + self.executable_arch + ' ###"\n' |
| 1157 |
|
txt += ' export SCRAM_ARCH='+self.executable_arch+'\n' |
| 1158 |
|
txt += ' export BUILD_ARCH='+self.executable_arch+'\n' |
| 1159 |
|
txt += ' if [ ! $VO_CMS_SW_DIR ] ;then\n' |
| 1160 |
< |
txt += ' echo "SET_CMS_ENV 10031 ==> ERROR CMS software dir not found on WN `hostname`"\n' |
| 1161 |
< |
txt += ' echo "JOB_EXIT_STATUS = 10031" \n' |
| 1162 |
< |
txt += ' echo "JobExitCode=10031" | tee -a $RUNTIME_AREA/$repo\n' |
| 1254 |
< |
txt += ' dumpStatus $RUNTIME_AREA/$repo\n' |
| 1255 |
< |
txt += ' exit 1\n' |
| 1160 |
> |
txt += ' echo "ERROR ==> CMS software dir not found on WN `hostname`"\n' |
| 1161 |
> |
txt += ' job_exit_code=10031\n' |
| 1162 |
> |
txt += ' func_exit\n' |
| 1163 |
|
txt += ' else\n' |
| 1164 |
|
txt += ' echo "Sourcing environment... "\n' |
| 1165 |
|
txt += ' if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n' |
| 1166 |
< |
txt += ' echo "SET_CMS_ENV 10020 ==> ERROR cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n' |
| 1167 |
< |
txt += ' echo "JOB_EXIT_STATUS = 10020"\n' |
| 1168 |
< |
txt += ' echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n' |
| 1262 |
< |
txt += ' dumpStatus $RUNTIME_AREA/$repo\n' |
| 1263 |
< |
txt += ' exit 1\n' |
| 1166 |
> |
txt += ' echo "ERROR ==> cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n' |
| 1167 |
> |
txt += ' job_exit_code=10020\n' |
| 1168 |
> |
txt += ' func_exit\n' |
| 1169 |
|
txt += ' fi\n' |
| 1170 |
|
txt += ' echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n' |
| 1171 |
|
txt += ' source $VO_CMS_SW_DIR/cmsset_default.sh\n' |
| 1172 |
|
txt += ' result=$?\n' |
| 1173 |
|
txt += ' if [ $result -ne 0 ]; then\n' |
| 1174 |
< |
txt += ' echo "SET_CMS_ENV 10032 ==> ERROR problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n' |
| 1175 |
< |
txt += ' echo "JOB_EXIT_STATUS = 10032"\n' |
| 1176 |
< |
txt += ' echo "JobExitCode=10032" | tee -a $RUNTIME_AREA/$repo\n' |
| 1272 |
< |
txt += ' dumpStatus $RUNTIME_AREA/$repo\n' |
| 1273 |
< |
txt += ' exit 1\n' |
| 1174 |
> |
txt += ' echo "ERROR ==> problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n' |
| 1175 |
> |
txt += ' job_exit_code=10032\n' |
| 1176 |
> |
txt += ' func_exit\n' |
| 1177 |
|
txt += ' fi\n' |
| 1178 |
|
txt += ' fi\n' |
| 1179 |
|
txt += ' \n' |
| 1180 |
< |
txt += ' echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n' |
| 1180 |
> |
txt += ' echo "==> setup cms environment ok"\n' |
| 1181 |
|
return txt |
| 1182 |
|
|
| 1183 |
|
### FEDE FOR DBS OUTPUT PUBLICATION |
| 1186 |
|
insert the part of the script that modifies the FrameworkJob Report |
| 1187 |
|
""" |
| 1188 |
|
|
| 1189 |
< |
txt = '' |
| 1190 |
< |
try: |
| 1288 |
< |
publish_data = int(self.cfg_params['USER.publish_data']) |
| 1289 |
< |
except KeyError: |
| 1290 |
< |
publish_data = 0 |
| 1189 |
> |
txt = '\n#Written by cms_cmssw::modifyReport\n' |
| 1190 |
> |
publish_data = int(self.cfg_params.get('USER.publish_data',0)) |
| 1191 |
|
if (publish_data == 1): |
| 1192 |
< |
txt += 'echo ">>> Modify Job Report:" \n' |
| 1193 |
< |
################ FEDE FOR DBS2 ############################################# |
| 1294 |
< |
#txt += 'chmod a+x $SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n' |
| 1295 |
< |
txt += 'chmod a+x $SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py\n' |
| 1296 |
< |
############################################################################# |
| 1192 |
> |
processedDataset = self.cfg_params['USER.publish_data_name'] |
| 1193 |
> |
LFNBaseName = LFNBase(processedDataset) |
| 1194 |
|
|
| 1195 |
< |
txt += 'if [ -z "$SE" ]; then\n' |
| 1196 |
< |
txt += ' SE="" \n' |
| 1197 |
< |
txt += 'fi \n' |
| 1198 |
< |
txt += 'if [ -z "$SE_PATH" ]; then\n' |
| 1199 |
< |
txt += ' SE_PATH="" \n' |
| 1200 |
< |
txt += 'fi \n' |
| 1201 |
< |
txt += 'echo "SE = $SE"\n' |
| 1305 |
< |
txt += 'echo "SE_PATH = $SE_PATH"\n' |
| 1195 |
> |
txt += 'if [ $copy_exit_status -eq 0 ]; then\n' |
| 1196 |
> |
txt += ' FOR_LFN=%s_${PSETHASH}/\n'%(LFNBaseName) |
| 1197 |
> |
txt += 'else\n' |
| 1198 |
> |
txt += ' FOR_LFN=/copy_problems/ \n' |
| 1199 |
> |
txt += ' SE=""\n' |
| 1200 |
> |
txt += ' SE_PATH=""\n' |
| 1201 |
> |
txt += 'fi\n' |
| 1202 |
|
|
| 1203 |
< |
processedDataset = self.cfg_params['USER.publish_data_name'] |
| 1203 |
> |
txt += 'echo ">>> Modify Job Report:" \n' |
| 1204 |
> |
txt += 'chmod a+x $SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py\n' |
| 1205 |
|
txt += 'ProcessedDataset='+processedDataset+'\n' |
| 1309 |
– |
#### LFN=/store/user/<user>/processedDataset_PSETHASH |
| 1310 |
– |
txt += 'if [ "$SE_PATH" == "" ]; then\n' |
| 1311 |
– |
#### FEDE: added slash in LFN ############## |
| 1312 |
– |
txt += ' FOR_LFN=/copy_problems/ \n' |
| 1313 |
– |
txt += 'else \n' |
| 1314 |
– |
txt += ' tmp=`echo $SE_PATH | awk -F \'store\' \'{print$2}\'` \n' |
| 1315 |
– |
##### FEDE TO BE CHANGED, BECAUSE STORE IS HARDCODED!!!! ######## |
| 1316 |
– |
txt += ' FOR_LFN=/store$tmp \n' |
| 1317 |
– |
txt += 'fi \n' |
| 1206 |
|
txt += 'echo "ProcessedDataset = $ProcessedDataset"\n' |
| 1207 |
+ |
txt += 'echo "SE = $SE"\n' |
| 1208 |
+ |
txt += 'echo "SE_PATH = $SE_PATH"\n' |
| 1209 |
|
txt += 'echo "FOR_LFN = $FOR_LFN" \n' |
| 1210 |
|
txt += 'echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n' |
| 1211 |
< |
txt += 'echo "$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n' |
| 1212 |
< |
txt += '$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n' |
| 1323 |
< |
|
| 1211 |
> |
txt += 'echo "$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n' |
| 1212 |
> |
txt += '$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n' |
| 1213 |
|
txt += 'modifyReport_result=$?\n' |
| 1325 |
– |
txt += 'echo modifyReport_result = $modifyReport_result\n' |
| 1214 |
|
txt += 'if [ $modifyReport_result -ne 0 ]; then\n' |
| 1215 |
< |
txt += ' exit_status=1\n' |
| 1216 |
< |
txt += ' echo "ERROR: Problem with ModifyJobReport"\n' |
| 1215 |
> |
txt += ' modifyReport_result=70500\n' |
| 1216 |
> |
txt += ' job_exit_code=$modifyReport_result\n' |
| 1217 |
> |
txt += ' echo "ModifyReportResult=$modifyReport_result" | tee -a $RUNTIME_AREA/$repo\n' |
| 1218 |
> |
txt += ' echo "WARNING: Problem with ModifyJobReport"\n' |
| 1219 |
|
txt += 'else\n' |
| 1220 |
< |
txt += ' mv NewFrameworkJobReport.xml crab_fjr_$NJob.xml\n' |
| 1220 |
> |
txt += ' mv NewFrameworkJobReport.xml $RUNTIME_AREA/crab_fjr_$NJob.xml\n' |
| 1221 |
|
txt += 'fi\n' |
| 1332 |
– |
else: |
| 1333 |
– |
txt += 'echo "no data publication required"\n' |
| 1334 |
– |
return txt |
| 1335 |
– |
|
| 1336 |
– |
def cleanEnv(self): |
| 1337 |
– |
txt = '' |
| 1338 |
– |
txt += 'if [ $middleware == OSG ]; then\n' |
| 1339 |
– |
txt += ' cd $RUNTIME_AREA\n' |
| 1340 |
– |
txt += ' echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n' |
| 1341 |
– |
txt += ' echo ">>> Remove working directory: $WORKING_DIR"\n' |
| 1342 |
– |
txt += ' /bin/rm -rf $WORKING_DIR\n' |
| 1343 |
– |
txt += ' if [ -d $WORKING_DIR ] ;then\n' |
| 1344 |
– |
txt += ' echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n' |
| 1345 |
– |
txt += ' echo "JOB_EXIT_STATUS = 60999"\n' |
| 1346 |
– |
txt += ' echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n' |
| 1347 |
– |
txt += ' dumpStatus $RUNTIME_AREA/$repo\n' |
| 1348 |
– |
txt += ' fi\n' |
| 1349 |
– |
txt += 'fi\n' |
| 1350 |
– |
txt += '\n' |
| 1222 |
|
return txt |
| 1223 |
|
|
| 1224 |
|
def setParam_(self, param, value): |
| 1236 |
|
nd[e]=0 |
| 1237 |
|
return nd.keys() |
| 1238 |
|
|
| 1239 |
< |
|
| 1369 |
< |
def checkOut(self, limit): |
| 1239 |
> |
def outList(self): |
| 1240 |
|
""" |
| 1241 |
|
check the dimension of the output files |
| 1242 |
|
""" |
| 1243 |
< |
txt = 'echo ">>> Starting output sandbox limit check :"\n' |
| 1244 |
< |
allOutFiles = "" |
| 1243 |
> |
txt = '' |
| 1244 |
> |
txt += 'echo ">>> list of expected files on output sandbox"\n' |
| 1245 |
|
listOutFiles = [] |
| 1246 |
< |
txt += 'stdoutFile=`ls *stdout` \n' |
| 1247 |
< |
txt += 'stderrFile=`ls *stderr` \n' |
| 1246 |
> |
stdout = 'CMSSW_$NJob.stdout' |
| 1247 |
> |
stderr = 'CMSSW_$NJob.stderr' |
| 1248 |
|
if (self.return_data == 1): |
| 1249 |
< |
for fileOut in (self.output_file+self.output_file_sandbox): |
| 1250 |
< |
allOutFiles = allOutFiles + " " + self.numberFile_(fileOut, '$NJob') + " $stdoutFile $stderrFile" |
| 1251 |
< |
else: |
| 1252 |
< |
for fileOut in (self.output_file_sandbox): |
| 1253 |
< |
txt += 'echo " '+fileOut+'";\n' |
| 1254 |
< |
allOutFiles = allOutFiles + " " + self.numberFile_(fileOut, '$NJob') + " $stdoutFile $stderrFile" |
| 1255 |
< |
txt += 'echo "OUTPUT files: '+str(allOutFiles)+'";\n' |
| 1256 |
< |
txt += 'ls -gGhrta;\n' |
| 1257 |
< |
txt += 'sum=0;\n' |
| 1258 |
< |
txt += 'for file in '+str(allOutFiles)+' ; do\n' |
| 1259 |
< |
txt += ' if [ -e $file ]; then\n' |
| 1260 |
< |
txt += ' tt=`ls -gGrta $file | awk \'{ print $3 }\'`\n' |
| 1391 |
< |
txt += ' sum=`expr $sum + $tt`\n' |
| 1392 |
< |
txt += ' else\n' |
| 1393 |
< |
txt += ' echo "WARNING: output file $file not found!"\n' |
| 1394 |
< |
txt += ' fi\n' |
| 1395 |
< |
txt += 'done\n' |
| 1396 |
< |
txt += 'echo "Total Output dimension: $sum";\n' |
| 1397 |
< |
txt += 'limit='+str(limit)+';\n' |
| 1398 |
< |
txt += 'echo "OUTPUT FILES LIMIT SET TO: $limit";\n' |
| 1399 |
< |
txt += 'if [ $limit -lt $sum ]; then\n' |
| 1400 |
< |
txt += ' echo "WARNING: output files have to big size - something will be lost;"\n' |
| 1401 |
< |
txt += ' echo " checking the output file sizes..."\n' |
| 1402 |
< |
txt += ' tot=0;\n' |
| 1403 |
< |
txt += ' for filefile in '+str(allOutFiles)+' ; do\n' |
| 1404 |
< |
txt += ' dimFile=`ls -gGrta $filefile | awk \'{ print $3 }\';`\n' |
| 1405 |
< |
txt += ' tot=`expr $tot + $tt`;\n' |
| 1406 |
< |
txt += ' if [ $limit -lt $dimFile ]; then\n' |
| 1407 |
< |
txt += ' echo "deleting file: $filefile";\n' |
| 1408 |
< |
txt += ' rm -f $filefile\n' |
| 1409 |
< |
txt += ' elif [ $limit -lt $tot ]; then\n' |
| 1410 |
< |
txt += ' echo "deleting file: $filefile";\n' |
| 1411 |
< |
txt += ' rm -f $filefile\n' |
| 1412 |
< |
txt += ' else\n' |
| 1413 |
< |
txt += ' echo "saving file: $filefile"\n' |
| 1414 |
< |
txt += ' fi\n' |
| 1415 |
< |
txt += ' done\n' |
| 1416 |
< |
|
| 1417 |
< |
txt += ' ls -agGhrt;\n' |
| 1418 |
< |
txt += ' echo "WARNING: output files are too big in dimension: can not put in the output_sandbox.";\n' |
| 1419 |
< |
txt += ' echo "JOB_EXIT_STATUS = 70000";\n' |
| 1420 |
< |
txt += ' exit_status=70000;\n' |
| 1421 |
< |
txt += 'else' |
| 1422 |
< |
txt += ' echo "Total Output dimension $sum is fine.";\n' |
| 1423 |
< |
txt += 'fi\n' |
| 1424 |
< |
txt += 'echo "Ending output sandbox limit check"\n' |
| 1249 |
> |
for file in (self.output_file+self.output_file_sandbox): |
| 1250 |
> |
listOutFiles.append(self.numberFile_(file, '$NJob')) |
| 1251 |
> |
listOutFiles.append(stdout) |
| 1252 |
> |
listOutFiles.append(stderr) |
| 1253 |
> |
else: |
| 1254 |
> |
for file in (self.output_file_sandbox): |
| 1255 |
> |
listOutFiles.append(self.numberFile_(file, '$NJob')) |
| 1256 |
> |
listOutFiles.append(stdout) |
| 1257 |
> |
listOutFiles.append(stderr) |
| 1258 |
> |
txt += 'echo "output files: '+string.join(listOutFiles,' ')+'"\n' |
| 1259 |
> |
txt += 'filesToCheck="'+string.join(listOutFiles,' ')+'"\n' |
| 1260 |
> |
txt += 'export filesToCheck\n' |
| 1261 |
|
return txt |
