| 34 |
|
self.executable = '' |
| 35 |
|
self.executable_arch = self.scram.getArch() |
| 36 |
|
self.tgz_name = 'default.tgz' |
| 37 |
– |
self.additional_tgz_name = 'additional.tgz' |
| 37 |
|
self.scriptName = 'CMSSW.sh' |
| 38 |
< |
self.pset = '' #scrip use case Da |
| 39 |
< |
self.datasetPath = '' #scrip use case Da |
| 38 |
> |
self.pset = '' |
| 39 |
> |
self.datasetPath = '' |
| 40 |
|
|
| 41 |
|
# set FJR file name |
| 42 |
|
self.fjrFileName = 'crab_fjr.xml' |
| 43 |
|
|
| 44 |
|
self.version = self.scram.getSWVersion() |
| 45 |
< |
|
| 46 |
< |
# |
| 47 |
< |
# Try to block creation in case of arch/version mismatch |
| 48 |
< |
# |
| 49 |
< |
|
| 50 |
< |
# a = string.split(self.version, "_") |
| 51 |
< |
# |
| 52 |
< |
# if int(a[1]) == 1 and (int(a[2]) < 5 and self.executable_arch.find('slc4') == 0): |
| 53 |
< |
# msg = "Warning: You are using %s version of CMSSW with %s architecture. \n--> Did you compile your libraries with SLC3? Otherwise you can find some problems running on SLC4 Grid nodes.\n"%(self.version, self.executable_arch) |
| 54 |
< |
# common.logger.message(msg) |
| 55 |
< |
# if int(a[1]) == 1 and (int(a[2]) >= 5 and self.executable_arch.find('slc3') == 0): |
| 57 |
< |
# msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch) |
| 58 |
< |
# raise CrabException(msg) |
| 59 |
< |
# |
| 60 |
< |
|
| 45 |
> |
version_array = self.version.split('_') |
| 46 |
> |
self.CMSSW_major = 0 |
| 47 |
> |
self.CMSSW_minor = 0 |
| 48 |
> |
self.CMSSW_patch = 0 |
| 49 |
> |
try: |
| 50 |
> |
self.CMSSW_major = int(version_array[1]) |
| 51 |
> |
self.CMSSW_minor = int(version_array[2]) |
| 52 |
> |
self.CMSSW_patch = int(version_array[3]) |
| 53 |
> |
except: |
| 54 |
> |
msg = "Cannot parse CMSSW version string: " + self.version + " for major and minor release number!" |
| 55 |
> |
raise CrabException(msg) |
| 56 |
|
|
| 57 |
|
### collect Data cards |
| 58 |
|
|
| 69 |
|
self.selectNoInput = 0 |
| 70 |
|
|
| 71 |
|
self.dataTiers = [] |
| 72 |
< |
|
| 72 |
> |
self.debugWrap = '' |
| 73 |
> |
self.debug_wrapper = cfg_params.get('USER.debug_wrapper',False) |
| 74 |
> |
if self.debug_wrapper: self.debugWrap='--debug' |
| 75 |
|
## now the application |
| 76 |
|
self.executable = cfg_params.get('CMSSW.executable','cmsRun') |
| 77 |
|
log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable) |
| 115 |
|
raise CrabException(msg) |
| 116 |
|
self.additional_inbox_files.append(string.strip(self.scriptExe)) |
| 117 |
|
|
| 121 |
– |
#CarlosDaniele |
| 118 |
|
if self.datasetPath == None and self.pset == None and self.scriptExe == '' : |
| 119 |
|
msg ="Error. script_exe not defined" |
| 120 |
|
raise CrabException(msg) |
| 137 |
|
if not os.path.exists(file): |
| 138 |
|
raise CrabException("Additional input file not found: "+file) |
| 139 |
|
pass |
| 144 |
– |
# fname = string.split(file, '/')[-1] |
| 145 |
– |
# storedFile = common.work_space.pathForTgz()+'share/'+fname |
| 146 |
– |
# shutil.copyfile(file, storedFile) |
| 140 |
|
self.additional_inbox_files.append(string.strip(file)) |
| 141 |
|
pass |
| 142 |
|
pass |
| 162 |
|
if cfg_params.has_key('CMSSW.total_number_of_events'): |
| 163 |
|
self.total_number_of_events = int(cfg_params['CMSSW.total_number_of_events']) |
| 164 |
|
self.selectTotalNumberEvents = 1 |
| 165 |
+ |
if self.selectNumberOfJobs == 1: |
| 166 |
+ |
if int(self.total_number_of_events) < int(self.theNumberOfJobs): |
| 167 |
+ |
msg = 'Must specify at least one event per job. total_number_of_events > number_of_jobs ' |
| 168 |
+ |
raise CrabException(msg) |
| 169 |
|
else: |
| 170 |
|
self.total_number_of_events = 0 |
| 171 |
|
self.selectTotalNumberEvents = 0 |
| 172 |
|
|
| 173 |
< |
if self.pset != None: #CarlosDaniele |
| 173 |
> |
if self.pset != None: |
| 174 |
|
if ( (self.selectTotalNumberEvents + self.selectEventsPerJob + self.selectNumberOfJobs) != 2 ): |
| 175 |
|
msg = 'Must define exactly two of total_number_of_events, events_per_job, or number_of_jobs.' |
| 176 |
|
raise CrabException(msg) |
| 200 |
|
if self.sourceSeed: |
| 201 |
|
print "pythia_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds." |
| 202 |
|
self.incrementSeeds.append('sourceSeed') |
| 203 |
+ |
self.incrementSeeds.append('theSource') |
| 204 |
|
|
| 205 |
|
self.sourceSeedVtx = cfg_params.get('CMSSW.vtx_seed',None) |
| 206 |
|
if self.sourceSeedVtx: |
| 240 |
|
blockSites = self.DataDiscoveryAndLocation(cfg_params) |
| 241 |
|
#DBSDLS-end |
| 242 |
|
|
| 245 |
– |
self.tgzNameWithPath = self.getTarBall(self.executable) |
| 243 |
|
|
| 244 |
|
## Select Splitting |
| 245 |
|
if self.selectNoInput: |
| 246 |
< |
if self.pset == None: #CarlosDaniele |
| 246 |
> |
if self.pset == None: |
| 247 |
|
self.jobSplittingForScript() |
| 248 |
|
else: |
| 249 |
|
self.jobSplittingNoInput() |
| 251 |
|
self.jobSplittingByBlocks(blockSites) |
| 252 |
|
|
| 253 |
|
# modify Pset |
| 254 |
< |
if self.pset != None: #CarlosDaniele |
| 254 |
> |
if self.pset != None: |
| 255 |
|
try: |
| 256 |
|
# Add FrameworkJobReport to parameter-set, set max events. |
| 257 |
|
# Reset later for data jobs by writeCFG which does all modifications |
| 258 |
< |
PsetEdit.addCrabFJR(self.fjrFileName) |
| 258 |
> |
PsetEdit.addCrabFJR(self.fjrFileName) # FUTURE: Job report addition not needed by CMSSW>1.5 |
| 259 |
|
PsetEdit.maxEvent(self.eventsPerJob) |
| 260 |
|
PsetEdit.psetWriter(self.configFilename()) |
| 261 |
|
except: |
| 262 |
< |
msg='Error while manipuliating ParameterSet: exiting...' |
| 262 |
> |
msg='Error while manipulating ParameterSet: exiting...' |
| 263 |
|
raise CrabException(msg) |
| 264 |
+ |
self.tgzNameWithPath = self.getTarBall(self.executable) |
| 265 |
|
|
| 266 |
|
def DataDiscoveryAndLocation(self, cfg_params): |
| 267 |
|
|
| 292 |
|
self.eventsbyfile=self.pubdata.getEventsPerFile() |
| 293 |
|
|
| 294 |
|
## get max number of events |
| 295 |
< |
self.maxEvents=self.pubdata.getMaxEvents() ## self.maxEvents used in Creator.py |
| 295 |
> |
self.maxEvents=self.pubdata.getMaxEvents() |
| 296 |
|
|
| 297 |
|
## Contact the DLS and build a list of sites hosting the fileblocks |
| 298 |
|
try: |
| 316 |
|
|
| 317 |
|
return sites |
| 318 |
|
|
| 321 |
– |
# to Be Removed DS -- BL |
| 322 |
– |
# def setArgsList(self, argsList): |
| 323 |
– |
# self.argsList = argsList |
| 324 |
– |
|
| 319 |
|
def jobSplittingByBlocks(self, blockSites): |
| 320 |
|
""" |
| 321 |
|
Perform job splitting. Jobs run over an integer number of files |
| 594 |
|
return |
| 595 |
|
|
| 596 |
|
|
| 597 |
< |
def jobSplittingForScript(self):#CarlosDaniele |
| 597 |
> |
def jobSplittingForScript(self): |
| 598 |
|
""" |
| 599 |
|
Perform job splitting based on number of job |
| 600 |
|
""" |
| 610 |
|
# argument is seed number.$i |
| 611 |
|
self.list_of_args = [] |
| 612 |
|
for i in range(self.total_number_of_jobs): |
| 619 |
– |
## Since there is no input, any site is good |
| 620 |
– |
# self.jobDestination.append(["Any"]) |
| 613 |
|
self.jobDestination.append([""]) |
| 622 |
– |
## no random seed |
| 614 |
|
self.list_of_args.append([str(i)]) |
| 615 |
|
return |
| 616 |
|
|
| 617 |
|
def split(self, jobParams): |
| 618 |
|
|
| 628 |
– |
#### Fabio |
| 619 |
|
njobs = self.total_number_of_jobs |
| 620 |
|
arglist = self.list_of_args |
| 621 |
|
# create the empty structure |
| 632 |
|
argu='' |
| 633 |
|
if len(jobParams[job]): |
| 634 |
|
argu += concString.join(jobParams[job] ) |
| 635 |
< |
job_ToSave['arguments']= str(job+1)+' '+argu## new BL--DS |
| 636 |
< |
job_ToSave['dlsDestination']= self.jobDestination[job]## new BL--DS |
| 647 |
< |
#common._db.updateJob_(job,job_ToSave)## new BL--DS |
| 635 |
> |
job_ToSave['arguments']= str(job+1)+' '+argu |
| 636 |
> |
job_ToSave['dlsDestination']= self.jobDestination[job] |
| 637 |
|
listField.append(job_ToSave) |
| 638 |
|
msg="Job "+str(job)+" Arguments: "+str(job+1)+" "+argu+"\n" \ |
| 639 |
|
+" Destination: "+str(self.jobDestination[job]) |
| 640 |
|
common.logger.debug(5,msg) |
| 641 |
< |
#common.logger.debug(5,"Job "+str(job)+" Destination: "+str(self.jobDestination[job])) |
| 642 |
< |
common._db.updateJob_(listID,listField)## new BL--DS |
| 654 |
< |
## Pay Attention Here....DS--BL |
| 655 |
< |
self.argsList = (len(jobParams[1])+1) |
| 641 |
> |
common._db.updateJob_(listID,listField) |
| 642 |
> |
self.argsList = (len(jobParams[0])+1) |
| 643 |
|
|
| 644 |
|
return |
| 645 |
|
|
| 646 |
|
def numberOfJobs(self): |
| 660 |
– |
# Fabio |
| 647 |
|
return self.total_number_of_jobs |
| 648 |
|
|
| 649 |
|
def getTarBall(self, exe): |
| 650 |
|
""" |
| 651 |
|
Return the TarBall with lib and exe |
| 652 |
|
""" |
| 667 |
– |
|
| 668 |
– |
# if it exist, just return it |
| 669 |
– |
# |
| 670 |
– |
# Marco. Let's start to use relative path for Boss XML files |
| 671 |
– |
# |
| 653 |
|
self.tgzNameWithPath = common.work_space.pathForTgz()+'share/'+self.tgz_name |
| 654 |
|
if os.path.exists(self.tgzNameWithPath): |
| 655 |
|
return self.tgzNameWithPath |
| 663 |
|
|
| 664 |
|
# First of all declare the user Scram area |
| 665 |
|
swArea = self.scram.getSWArea_() |
| 685 |
– |
#print "swArea = ", swArea |
| 686 |
– |
# swVersion = self.scram.getSWVersion() |
| 687 |
– |
# print "swVersion = ", swVersion |
| 666 |
|
swReleaseTop = self.scram.getReleaseTop_() |
| 689 |
– |
#print "swReleaseTop = ", swReleaseTop |
| 667 |
|
|
| 668 |
|
## check if working area is release top |
| 669 |
|
if swReleaseTop == '' or swArea == swReleaseTop: |
| 710 |
|
|
| 711 |
|
## Now check if any data dir(s) is present |
| 712 |
|
swAreaLen=len(swArea) |
| 713 |
+ |
self.dataExist = False |
| 714 |
|
for root, dirs, files in os.walk(swArea): |
| 715 |
|
if "data" in dirs: |
| 716 |
+ |
self.dataExist=True |
| 717 |
|
common.logger.debug(5,"data "+root+"/data"+" to be tarred") |
| 718 |
|
tar.add(root+"/data",root[swAreaLen:]+"/data") |
| 719 |
|
|
| 720 |
+ |
### CMSSW ParameterSet |
| 721 |
+ |
if not self.pset is None: |
| 722 |
+ |
cfg_file = common.work_space.jobDir()+self.configFilename() |
| 723 |
+ |
tar.add(cfg_file,self.configFilename()) |
| 724 |
+ |
common.logger.debug(5,"File added to "+self.tgzNameWithPath+" : "+str(tar.getnames())) |
| 725 |
+ |
|
| 726 |
|
|
| 727 |
|
## Add ProdCommon dir to tar |
| 728 |
|
prodcommonDir = 'ProdCommon' |
| 729 |
|
prodcommonPath = os.environ['CRABDIR'] + '/' + 'ProdCommon' |
| 730 |
|
if os.path.isdir(prodcommonPath): |
| 731 |
|
tar.add(prodcommonPath,prodcommonDir) |
| 732 |
+ |
common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames())) |
| 733 |
|
|
| 734 |
+ |
##### ML stuff |
| 735 |
+ |
ML_file_list=['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py'] |
| 736 |
+ |
path=os.environ['CRABDIR'] + '/python/' |
| 737 |
+ |
for file in ML_file_list: |
| 738 |
+ |
tar.add(path+file,file) |
| 739 |
+ |
common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames())) |
| 740 |
+ |
|
| 741 |
+ |
##### Utils |
| 742 |
+ |
Utils_file_list=['parseCrabFjr.py','writeCfg.py', 'JobReportErrorCode.py'] |
| 743 |
+ |
for file in Utils_file_list: |
| 744 |
+ |
tar.add(path+file,file) |
| 745 |
+ |
common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames())) |
| 746 |
+ |
|
| 747 |
+ |
##### AdditionalFiles |
| 748 |
+ |
for file in self.additional_inbox_files: |
| 749 |
+ |
tar.add(file,string.split(file,'/')[-1]) |
| 750 |
|
common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames())) |
| 751 |
+ |
|
| 752 |
|
tar.close() |
| 753 |
|
except : |
| 754 |
|
raise CrabException('Could not create tar-ball') |
| 759 |
|
raise CrabException('Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' + str(self.MaxTarBallSize) + ' MB input sandbox limit and not supported by the used GRID submission system. Please make sure that no unnecessary files are in all data directories in your local CMSSW project area as they are automatically packed into the input sandbox.') |
| 760 |
|
|
| 761 |
|
## create tar-ball with ML stuff |
| 759 |
– |
self.MLtgzfile = common.work_space.pathForTgz()+'share/MLfiles.tgz' |
| 760 |
– |
try: |
| 761 |
– |
tar = tarfile.open(self.MLtgzfile, "w:gz") |
| 762 |
– |
path=os.environ['CRABDIR'] + '/python/' |
| 763 |
– |
for file in ['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py', 'parseCrabFjr.py','writeCfg.py', 'JobReportErrorCode.py']: |
| 764 |
– |
tar.add(path+file,file) |
| 765 |
– |
common.logger.debug(5,"Files added to "+self.MLtgzfile+" : "+str(tar.getnames())) |
| 766 |
– |
tar.close() |
| 767 |
– |
except : |
| 768 |
– |
raise CrabException('Could not create ML files tar-ball') |
| 769 |
– |
|
| 770 |
– |
return |
| 771 |
– |
|
| 772 |
– |
def additionalInputFileTgz(self): |
| 773 |
– |
""" |
| 774 |
– |
Put all additional files into a tar ball and return its name |
| 775 |
– |
""" |
| 776 |
– |
import tarfile |
| 777 |
– |
tarName= common.work_space.pathForTgz()+'share/'+self.additional_tgz_name |
| 778 |
– |
tar = tarfile.open(tarName, "w:gz") |
| 779 |
– |
for file in self.additional_inbox_files: |
| 780 |
– |
tar.add(file,string.split(file,'/')[-1]) |
| 781 |
– |
common.logger.debug(5,"Files added to "+self.additional_tgz_name+" : "+str(tar.getnames())) |
| 782 |
– |
tar.close() |
| 783 |
– |
return tarName |
| 762 |
|
|
| 763 |
|
def wsSetupEnvironment(self, nj=0): |
| 764 |
|
""" |
| 765 |
|
Returns part of a job script which prepares |
| 766 |
|
the execution environment for the job 'nj'. |
| 767 |
|
""" |
| 768 |
+ |
if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3): |
| 769 |
+ |
psetName = 'pset.py' |
| 770 |
+ |
else: |
| 771 |
+ |
psetName = 'pset.cfg' |
| 772 |
|
# Prepare JobType-independent part |
| 773 |
|
txt = '\n#Written by cms_cmssw::wsSetupEnvironment\n' |
| 774 |
|
txt += 'echo ">>> setup environment"\n' |
| 787 |
|
txt += ' cd $WORKING_DIR\n' |
| 788 |
|
txt += ' echo ">>> current directory (WORKING_DIR): $WORKING_DIR"\n' |
| 789 |
|
txt += self.wsSetupCMSOSGEnvironment_() |
| 808 |
– |
#txt += ' echo "### Set SCRAM ARCH to ' + self.executable_arch + ' ###"\n' |
| 809 |
– |
#txt += ' export SCRAM_ARCH='+self.executable_arch+'\n' |
| 790 |
|
txt += 'fi\n' |
| 791 |
|
|
| 792 |
|
# Prepare JobType-specific part |
| 802 |
|
txt += ' func_exit\n' |
| 803 |
|
txt += 'fi \n' |
| 804 |
|
txt += 'cd '+self.version+'\n' |
| 825 |
– |
########## FEDE FOR DBS2 ###################### |
| 805 |
|
txt += 'SOFTWARE_DIR=`pwd`\n' |
| 806 |
|
txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n' |
| 828 |
– |
############################################### |
| 829 |
– |
### needed grep for bug in scramv1 ### |
| 807 |
|
txt += 'eval `'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME`\n' |
| 808 |
+ |
txt += 'if [ $? != 0 ] ; then\n' |
| 809 |
+ |
txt += ' echo "ERROR ==> Problem with the command: "\n' |
| 810 |
+ |
txt += ' echo "eval \`'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME \` at `hostname`"\n' |
| 811 |
+ |
txt += ' job_exit_code=10034\n' |
| 812 |
+ |
txt += ' func_exit\n' |
| 813 |
+ |
txt += 'fi \n' |
| 814 |
|
# Handle the arguments: |
| 815 |
|
txt += "\n" |
| 816 |
|
txt += "## number of arguments (first argument always jobnumber)\n" |
| 825 |
|
|
| 826 |
|
# Prepare job-specific part |
| 827 |
|
job = common.job_list[nj] |
| 845 |
– |
### FEDE FOR DBS OUTPUT PUBLICATION |
| 828 |
|
if (self.datasetPath): |
| 829 |
|
txt += '\n' |
| 830 |
|
txt += 'DatasetPath='+self.datasetPath+'\n' |
| 860 |
|
txt += 'FirstRun=${args[1]}; export FirstRun\n' |
| 861 |
|
txt += 'echo "FirstRun: <$FirstRun>"\n' |
| 862 |
|
|
| 863 |
< |
txt += 'mv -f '+pset+' pset.cfg\n' |
| 863 |
> |
txt += 'mv -f ' + pset + ' ' + psetName + '\n' |
| 864 |
|
|
| 883 |
– |
if len(self.additional_inbox_files) > 0: |
| 884 |
– |
txt += 'if [ -e $RUNTIME_AREA/'+self.additional_tgz_name+' ] ; then\n' |
| 885 |
– |
txt += ' tar xzvf $RUNTIME_AREA/'+self.additional_tgz_name+'\n' |
| 886 |
– |
txt += 'fi\n' |
| 887 |
– |
pass |
| 865 |
|
|
| 866 |
|
if self.pset != None: |
| 867 |
+ |
# FUTURE: Can simply for 2_1_x and higher |
| 868 |
|
txt += '\n' |
| 869 |
< |
txt += 'echo "***** cat pset.cfg *********"\n' |
| 870 |
< |
txt += 'cat pset.cfg\n' |
| 871 |
< |
txt += 'echo "****** end pset.cfg ********"\n' |
| 872 |
< |
txt += '\n' |
| 873 |
< |
txt += 'PSETHASH=`EdmConfigHash < pset.cfg` \n' |
| 869 |
> |
if self.debug_wrapper==True: |
| 870 |
> |
txt += 'echo "***** cat ' + psetName + ' *********"\n' |
| 871 |
> |
txt += 'cat ' + psetName + '\n' |
| 872 |
> |
txt += 'echo "****** end ' + psetName + ' ********"\n' |
| 873 |
> |
txt += '\n' |
| 874 |
> |
txt += 'PSETHASH=`edmConfigHash < ' + psetName + '` \n' |
| 875 |
|
txt += 'echo "PSETHASH = $PSETHASH" \n' |
| 876 |
|
txt += '\n' |
| 877 |
|
return txt |
| 887 |
|
if os.path.isfile(self.tgzNameWithPath): |
| 888 |
|
txt += 'echo ">>> tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+' :" \n' |
| 889 |
|
txt += 'tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n' |
| 890 |
+ |
txt += 'ls -Al \n' |
| 891 |
|
txt += 'untar_status=$? \n' |
| 892 |
|
txt += 'if [ $untar_status -ne 0 ]; then \n' |
| 893 |
|
txt += ' echo "ERROR ==> Untarring .tgz file failed"\n' |
| 922 |
|
txt += 'rm -r lib/ module/ \n' |
| 923 |
|
txt += 'mv $RUNTIME_AREA/lib/ . \n' |
| 924 |
|
txt += 'mv $RUNTIME_AREA/module/ . \n' |
| 925 |
+ |
if self.dataExist == True: |
| 926 |
+ |
txt += 'rm -r src/ \n' |
| 927 |
+ |
txt += 'mv $RUNTIME_AREA/src/ . \n' |
| 928 |
+ |
if len(self.additional_inbox_files)>0: |
| 929 |
+ |
for file in self.additional_inbox_files: |
| 930 |
+ |
txt += 'mv $RUNTIME_AREA/'+os.path.basename(file)+' . \n' |
| 931 |
|
txt += 'mv $RUNTIME_AREA/ProdCommon/ . \n' |
| 932 |
|
|
| 933 |
|
txt += 'if [ -z "$PYTHONPATH" ]; then\n' |
| 947 |
|
""" |
| 948 |
|
|
| 949 |
|
def executableName(self): |
| 950 |
< |
if self.scriptExe: #CarlosDaniele |
| 950 |
> |
if self.scriptExe: |
| 951 |
|
return "sh " |
| 952 |
|
else: |
| 953 |
|
return self.executable |
| 957 |
|
if self.scriptExe:#CarlosDaniele |
| 958 |
|
return self.scriptExe + " $NJob" |
| 959 |
|
else: |
| 974 |
– |
version_array = self.scram.getSWVersion().split('_') |
| 975 |
– |
major = 0 |
| 976 |
– |
minor = 0 |
| 977 |
– |
try: |
| 978 |
– |
major = int(version_array[1]) |
| 979 |
– |
minor = int(version_array[2]) |
| 980 |
– |
except: |
| 981 |
– |
msg = "Cannot parse CMSSW version string: " + "_".join(version_array) + " for major and minor release number!" |
| 982 |
– |
raise CrabException(msg) |
| 983 |
– |
|
| 960 |
|
ex_args = "" |
| 961 |
|
# FUTURE: This tests the CMSSW version. Can remove code as versions deprecated |
| 962 |
|
# Framework job report |
| 963 |
< |
if major >= 1 and minor >= 5 : |
| 963 |
> |
if (self.CMSSW_major >= 1 and self.CMSSW_minor >= 5) or (self.CMSSW_major >= 2): |
| 964 |
|
ex_args += " -j $RUNTIME_AREA/crab_fjr_$NJob.xml" |
| 965 |
< |
# Type of cfg file |
| 966 |
< |
if major >= 2 : |
| 965 |
> |
# Type of config file |
| 966 |
> |
if self.CMSSW_major >= 2 : |
| 967 |
|
ex_args += " -p pset.py" |
| 968 |
|
else: |
| 969 |
|
ex_args += " -p pset.cfg" |
| 974 |
|
Returns a list of filenames to be put in JDL input sandbox. |
| 975 |
|
""" |
| 976 |
|
inp_box = [] |
| 1001 |
– |
# # dict added to delete duplicate from input sandbox file list |
| 1002 |
– |
# seen = {} |
| 1003 |
– |
## code |
| 977 |
|
if os.path.isfile(self.tgzNameWithPath): |
| 978 |
|
inp_box.append(self.tgzNameWithPath) |
| 1006 |
– |
if os.path.isfile(self.MLtgzfile): |
| 1007 |
– |
inp_box.append(self.MLtgzfile) |
| 1008 |
– |
## config |
| 1009 |
– |
if not self.pset is None: |
| 1010 |
– |
inp_box.append(common.work_space.pathForTgz() + 'job/' + self.configFilename()) |
| 1011 |
– |
## additional input files |
| 1012 |
– |
tgz = self.additionalInputFileTgz() |
| 1013 |
– |
inp_box.append(tgz) |
| 1014 |
– |
## executable |
| 979 |
|
wrapper = os.path.basename(str(common._db.queryTask('scriptName'))) |
| 980 |
|
inp_box.append(common.work_space.pathForTgz() +'job/'+ wrapper) |
| 981 |
|
return inp_box |
| 1070 |
|
req='Member("VO-cms-' + \ |
| 1071 |
|
self.version + \ |
| 1072 |
|
'", other.GlueHostApplicationSoftwareRunTimeEnvironment)' |
| 1073 |
< |
## SL add requirement for OS version only if SL4 |
| 1110 |
< |
#reSL4 = re.compile( r'slc4' ) |
| 1111 |
< |
if self.executable_arch: # and reSL4.search(self.executable_arch): |
| 1073 |
> |
if self.executable_arch: |
| 1074 |
|
req+=' && Member("VO-cms-' + \ |
| 1075 |
|
self.executable_arch + \ |
| 1076 |
|
'", other.GlueHostApplicationSoftwareRunTimeEnvironment)' |
| 1083 |
|
|
| 1084 |
|
def configFilename(self): |
| 1085 |
|
""" return the config filename """ |
| 1086 |
< |
return self.name()+'.cfg' |
| 1086 |
> |
# FUTURE: Can remove cfg mode for CMSSW >= 2_1_x |
| 1087 |
> |
if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3): |
| 1088 |
> |
return self.name()+'.py' |
| 1089 |
> |
else: |
| 1090 |
> |
return self.name()+'.cfg' |
| 1091 |
|
|
| 1092 |
|
def wsSetupCMSOSGEnvironment_(self): |
| 1093 |
|
""" |
| 1113 |
|
|
| 1114 |
|
return txt |
| 1115 |
|
|
| 1150 |
– |
### OLI_DANIELE |
| 1116 |
|
def wsSetupCMSLCGEnvironment_(self): |
| 1117 |
|
""" |
| 1118 |
|
Returns part of a job script which is prepares |
| 1147 |
|
txt += ' echo "==> setup cms environment ok"\n' |
| 1148 |
|
return txt |
| 1149 |
|
|
| 1185 |
– |
### FEDE FOR DBS OUTPUT PUBLICATION |
| 1150 |
|
def modifyReport(self, nj): |
| 1151 |
|
""" |
| 1152 |
|
insert the part of the script that modifies the FrameworkJob Report |
| 1153 |
|
""" |
| 1190 |
– |
|
| 1154 |
|
txt = '\n#Written by cms_cmssw::modifyReport\n' |
| 1155 |
|
publish_data = int(self.cfg_params.get('USER.publish_data',0)) |
| 1156 |
|
if (publish_data == 1): |
| 1164 |
|
txt += ' SE=""\n' |
| 1165 |
|
txt += ' SE_PATH=""\n' |
| 1166 |
|
txt += 'fi\n' |
| 1167 |
< |
|
| 1167 |
> |
|
| 1168 |
|
txt += 'echo ">>> Modify Job Report:" \n' |
| 1169 |
|
txt += 'chmod a+x $SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py\n' |
| 1170 |
|
txt += 'ProcessedDataset='+processedDataset+'\n' |
| 1186 |
|
txt += 'fi\n' |
| 1187 |
|
return txt |
| 1188 |
|
|
| 1189 |
+ |
def wsParseFJR(self): |
| 1190 |
+ |
""" |
| 1191 |
+ |
Parse the FrameworkJobReport to obtain useful infos |
| 1192 |
+ |
""" |
| 1193 |
+ |
txt = '\n#Written by cms_cmssw::wsParseFJR\n' |
| 1194 |
+ |
txt += 'echo ">>> Parse FrameworkJobReport crab_fjr.xml"\n' |
| 1195 |
+ |
txt += 'if [ -s $RUNTIME_AREA/crab_fjr_$NJob.xml ]; then\n' |
| 1196 |
+ |
txt += ' if [ -s $RUNTIME_AREA/parseCrabFjr.py ]; then\n' |
| 1197 |
+ |
txt += ' cmd_out=`python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --dashboard $MonitorID,$MonitorJobID '+self.debugWrap+'`\n' |
| 1198 |
+ |
if self.debug_wrapper : |
| 1199 |
+ |
txt += ' echo "Result of parsing the FrameworkJobReport crab_fjr.xml: $cmd_out"\n' |
| 1200 |
+ |
txt += ' executable_exit_status=`python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --exitcode`\n' |
| 1201 |
+ |
txt += ' if [ $executable_exit_status -eq 50115 ];then\n' |
| 1202 |
+ |
txt += ' echo ">>> crab_fjr.xml contents: "\n' |
| 1203 |
+ |
txt += ' cat $RUNTIME_AREA/crab_fjr_NJob.xml\n' |
| 1204 |
+ |
txt += ' echo "Wrong FrameworkJobReport --> does not contain useful info. ExitStatus: $executable_exit_status"\n' |
| 1205 |
+ |
txt += ' elif [ $executable_exit_status -eq -999 ];then\n' |
| 1206 |
+ |
txt += ' echo "ExitStatus from FrameworkJobReport not available. not available. Using exit code of executable from command line."\n' |
| 1207 |
+ |
txt += ' else\n' |
| 1208 |
+ |
txt += ' echo "Extracted ExitStatus from FrameworkJobReport parsing output: $executable_exit_status"\n' |
| 1209 |
+ |
txt += ' fi\n' |
| 1210 |
+ |
txt += ' else\n' |
| 1211 |
+ |
txt += ' echo "CRAB python script to parse CRAB FrameworkJobReport crab_fjr.xml is not available, using exit code of executable from command line."\n' |
| 1212 |
+ |
txt += ' fi\n' |
| 1213 |
+ |
#### Patch to check input data reading for CMSSW16x Hopefully we-ll remove it asap |
| 1214 |
+ |
|
| 1215 |
+ |
if self.datasetPath: |
| 1216 |
+ |
# VERIFY PROCESSED DATA |
| 1217 |
+ |
txt += ' if [ $executable_exit_status -eq 0 ];then\n' |
| 1218 |
+ |
txt += ' echo ">>> Verify list of processed files:"\n' |
| 1219 |
+ |
txt += ' echo $InputFiles |tr -d \'\\\\\' |tr \',\' \'\\n\'|tr -d \'"\' > input-files.txt\n' |
| 1220 |
+ |
txt += ' `python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --lfn` > processed-files.txt\n' |
| 1221 |
+ |
txt += ' cat input-files.txt | sort | uniq > tmp.txt\n' |
| 1222 |
+ |
txt += ' mv tmp.txt input-files.txt\n' |
| 1223 |
+ |
txt += ' echo "cat input-files.txt"\n' |
| 1224 |
+ |
txt += ' echo "----------------------"\n' |
| 1225 |
+ |
txt += ' cat input-files.txt\n' |
| 1226 |
+ |
txt += ' cat processed-files.txt | sort | uniq > tmp.txt\n' |
| 1227 |
+ |
txt += ' mv tmp.txt processed-files.txt\n' |
| 1228 |
+ |
txt += ' echo "----------------------"\n' |
| 1229 |
+ |
txt += ' echo "cat processed-files.txt"\n' |
| 1230 |
+ |
txt += ' echo "----------------------"\n' |
| 1231 |
+ |
txt += ' cat processed-files.txt\n' |
| 1232 |
+ |
txt += ' echo "----------------------"\n' |
| 1233 |
+ |
txt += ' diff -q input-files.txt processed-files.txt\n' |
| 1234 |
+ |
txt += ' fileverify_status=$?\n' |
| 1235 |
+ |
txt += ' if [ $fileverify_status -ne 0 ]; then\n' |
| 1236 |
+ |
txt += ' executable_exit_status=30001\n' |
| 1237 |
+ |
txt += ' echo "ERROR ==> not all input files processed"\n' |
| 1238 |
+ |
txt += ' echo " ==> list of processed files from crab_fjr.xml differs from list in pset.cfg"\n' |
| 1239 |
+ |
txt += ' echo " ==> diff input-files.txt processed-files.txt"\n' |
| 1240 |
+ |
txt += ' fi\n' |
| 1241 |
+ |
txt += ' fi\n' |
| 1242 |
+ |
txt += '\n' |
| 1243 |
+ |
txt += 'else\n' |
| 1244 |
+ |
txt += ' echo "CRAB FrameworkJobReport crab_fjr.xml is not available, using exit code of executable from command line."\n' |
| 1245 |
+ |
txt += 'fi\n' |
| 1246 |
+ |
txt += '\n' |
| 1247 |
+ |
txt += 'echo "ExeExitCode=$executable_exit_status" | tee -a $RUNTIME_AREA/$repo\n' |
| 1248 |
+ |
txt += 'echo "EXECUTABLE_EXIT_STATUS = $executable_exit_status"\n' |
| 1249 |
+ |
txt += 'job_exit_code=$executable_exit_status\n' |
| 1250 |
+ |
|
| 1251 |
+ |
return txt |
| 1252 |
+ |
|
| 1253 |
|
def setParam_(self, param, value): |
| 1254 |
|
self._params[param] = value |
| 1255 |
|
|
