ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/cms_cmssw.py
Revision: 1.298
Committed: Sun May 10 13:44:11 2009 UTC (15 years, 11 months ago) by spiga
Content type: text/x-python
Branch: MAIN
CVS Tags: CRAB_2_6_0_pre2
Changes since 1.297: +3 -2 lines
Log Message:
WMCore/__init__.py needed in the ISB

File Contents

# User Rev Content
1 slacapra 1.1 from JobType import JobType
2     from crab_logger import Logger
3     from crab_exceptions import *
4     from crab_util import *
5     import common
6     import Scram
7 spiga 1.269 from Splitter import JobSplitter
8 slacapra 1.1
9 spiga 1.293 from IMProv.IMProvNode import IMProvNode
10 slacapra 1.105 import os, string, glob
11 slacapra 1.1
12     class Cmssw(JobType):
13 spiga 1.208 def __init__(self, cfg_params, ncjobs,skip_blocks, isNew):
14 slacapra 1.1 JobType.__init__(self, 'CMSSW')
15     common.logger.debug(3,'CMSSW::__init__')
16 spiga 1.208 self.skip_blocks = skip_blocks
17 spiga 1.296 self.argsList = 1
18 mcinquil 1.144
19 gutsche 1.3 self._params = {}
20     self.cfg_params = cfg_params
21 ewv 1.254
22 spiga 1.234 ### Temporary patch to automatically skip the ISB size check:
23     server=self.cfg_params.get('CRAB.server_name',None)
24 ewv 1.250 size = 9.5
25 spiga 1.249 if server or common.scheduler.name().upper() in ['LSF','CAF']: size = 99999
26 spiga 1.234 ### D.S.
27     self.MaxTarBallSize = float(self.cfg_params.get('EDG.maxtarballsize',size))
28 gutsche 1.72
29 gutsche 1.44 # number of jobs requested to be created, limit obj splitting
30 gutsche 1.38 self.ncjobs = ncjobs
31    
32 slacapra 1.1 log = common.logger
33 ewv 1.131
34 slacapra 1.1 self.scram = Scram.Scram(cfg_params)
35     self.additional_inbox_files = []
36     self.scriptExe = ''
37     self.executable = ''
38 slacapra 1.71 self.executable_arch = self.scram.getArch()
39 slacapra 1.1 self.tgz_name = 'default.tgz'
40 corvo 1.56 self.scriptName = 'CMSSW.sh'
41 ewv 1.192 self.pset = ''
42 spiga 1.187 self.datasetPath = ''
43 gutsche 1.3
44 gutsche 1.50 # set FJR file name
45     self.fjrFileName = 'crab_fjr.xml'
46    
47 slacapra 1.1 self.version = self.scram.getSWVersion()
48 slacapra 1.275 common.logger.write("CMSSW version is: "+str(self.version))
49 ewv 1.182 try:
50 slacapra 1.291 type, self.CMSSW_major, self.CMSSW_minor, self.CMSSW_patch = tuple(self.version.split('_'))
51 ewv 1.182 except:
52 ewv 1.184 msg = "Cannot parse CMSSW version string: " + self.version + " for major and minor release number!"
53 ewv 1.182 raise CrabException(msg)
54    
55 ewv 1.276 if self.CMSSW_major < 1 or (self.CMSSW_major == 1 and self.CMSSW_minor < 5):
56     msg = "CRAB supports CMSSW >= 1_5_x only. Use an older CRAB version."
57     raise CrabException(msg)
58     """
59     As CMSSW versions are dropped we can drop more code:
60     1.X dropped: drop support for running .cfg on WN
61     2.0 dropped: drop all support for cfg here and in writeCfg
62     2.0 dropped: Recheck the random number seed support
63     """
64    
65 slacapra 1.1 ### collect Data cards
66 gutsche 1.66
67 ewv 1.226
68 fanzago 1.221 ### Temporary: added to remove input file control in the case of PU
69 farinafa 1.224 self.dataset_pu = cfg_params.get('CMSSW.dataset_pu', None)
70 ewv 1.226
71 slacapra 1.153 tmp = cfg_params['CMSSW.datasetpath']
72     log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
73 spiga 1.236
74     if tmp =='':
75     msg = "Error: datasetpath not defined "
76     raise CrabException(msg)
77     elif string.lower(tmp)=='none':
78 slacapra 1.153 self.datasetPath = None
79     self.selectNoInput = 1
80     else:
81     self.datasetPath = tmp
82     self.selectNoInput = 0
83 gutsche 1.5
84 slacapra 1.1 self.dataTiers = []
85 ewv 1.295
86 spiga 1.288 self.debugWrap=''
87 fanzago 1.285 self.debug_wrapper = int(cfg_params.get('USER.debug_wrapper',0))
88     if self.debug_wrapper == 1: self.debugWrap='--debug'
89 slacapra 1.291
90 slacapra 1.1 ## now the application
91 ewv 1.262 self.managedGenerators = ['madgraph','comphep']
92 ewv 1.258 self.generator = cfg_params.get('CMSSW.generator','pythia').lower()
93 slacapra 1.153 self.executable = cfg_params.get('CMSSW.executable','cmsRun')
94     log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable)
95 slacapra 1.1
96 slacapra 1.153 if not cfg_params.has_key('CMSSW.pset'):
97 slacapra 1.1 raise CrabException("PSet file missing. Cannot run cmsRun ")
98 slacapra 1.153 self.pset = cfg_params['CMSSW.pset']
99     log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset)
100     if self.pset.lower() != 'none' :
101     if (not os.path.exists(self.pset)):
102     raise CrabException("User defined PSet file "+self.pset+" does not exist")
103     else:
104     self.pset = None
105 slacapra 1.1
106     # output files
107 slacapra 1.53 ## stuff which must be returned always via sandbox
108     self.output_file_sandbox = []
109    
110     # add fjr report by default via sandbox
111     self.output_file_sandbox.append(self.fjrFileName)
112    
113     # other output files to be returned via sandbox or copied to SE
114 mcinquil 1.216 outfileflag = False
115 slacapra 1.153 self.output_file = []
116     tmp = cfg_params.get('CMSSW.output_file',None)
117     if tmp :
118 slacapra 1.207 self.output_file = [x.strip() for x in tmp.split(',')]
119 mcinquil 1.216 outfileflag = True #output found
120     #else:
121     # log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
122 slacapra 1.1
123     # script_exe file as additional file in inputSandbox
124 slacapra 1.153 self.scriptExe = cfg_params.get('USER.script_exe',None)
125     if self.scriptExe :
126 slacapra 1.176 if not os.path.isfile(self.scriptExe):
127     msg ="ERROR. file "+self.scriptExe+" not found"
128     raise CrabException(msg)
129     self.additional_inbox_files.append(string.strip(self.scriptExe))
130 slacapra 1.70
131 spiga 1.42 if self.datasetPath == None and self.pset == None and self.scriptExe == '' :
132 slacapra 1.176 msg ="Error. script_exe not defined"
133     raise CrabException(msg)
134 spiga 1.42
135 ewv 1.226 # use parent files...
136 spiga 1.269 self.useParent = int(self.cfg_params.get('CMSSW.use_parent',0))
137 spiga 1.204
138 slacapra 1.1 ## additional input files
139 slacapra 1.153 if cfg_params.has_key('USER.additional_input_files'):
140 slacapra 1.29 tmpAddFiles = string.split(cfg_params['USER.additional_input_files'],',')
141 slacapra 1.70 for tmp in tmpAddFiles:
142     tmp = string.strip(tmp)
143     dirname = ''
144     if not tmp[0]=="/": dirname = "."
145 corvo 1.85 files = []
146     if string.find(tmp,"*")>-1:
147     files = glob.glob(os.path.join(dirname, tmp))
148     if len(files)==0:
149     raise CrabException("No additional input file found with this pattern: "+tmp)
150     else:
151     files.append(tmp)
152 slacapra 1.70 for file in files:
153     if not os.path.exists(file):
154     raise CrabException("Additional input file not found: "+file)
155 slacapra 1.45 pass
156 slacapra 1.105 self.additional_inbox_files.append(string.strip(file))
157 slacapra 1.1 pass
158     pass
159 slacapra 1.70 common.logger.debug(5,"Additional input files: "+str(self.additional_inbox_files))
160 slacapra 1.153 pass
161 gutsche 1.3
162 gutsche 1.35
163 ewv 1.160 ## New method of dealing with seeds
164     self.incrementSeeds = []
165     self.preserveSeeds = []
166     if cfg_params.has_key('CMSSW.preserve_seeds'):
167     tmpList = cfg_params['CMSSW.preserve_seeds'].split(',')
168     for tmp in tmpList:
169     tmp.strip()
170     self.preserveSeeds.append(tmp)
171     if cfg_params.has_key('CMSSW.increment_seeds'):
172     tmpList = cfg_params['CMSSW.increment_seeds'].split(',')
173     for tmp in tmpList:
174     tmp.strip()
175     self.incrementSeeds.append(tmp)
176    
177 slacapra 1.153 self.firstRun = cfg_params.get('CMSSW.first_run',None)
178 slacapra 1.90
179 ewv 1.147 # Copy/return
180 slacapra 1.153 self.copy_data = int(cfg_params.get('USER.copy_data',0))
181     self.return_data = int(cfg_params.get('USER.return_data',0))
182 ewv 1.276
183     self.conf = {}
184     self.conf['pubdata'] = None
185 spiga 1.269 # number of jobs requested to be created, limit obj splitting DD
186 slacapra 1.1 #DBSDLS-start
187 ewv 1.131 ## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
188 slacapra 1.1 self.maxEvents=0 # max events available ( --> check the requested nb. of evts in Creator.py)
189     self.DBSPaths={} # all dbs paths requested ( --> input to the site local discovery script)
190 gutsche 1.35 self.jobDestination=[] # Site destination(s) for each job (list of lists)
191 slacapra 1.1 ## Perform the data location and discovery (based on DBS/DLS)
192 slacapra 1.9 ## SL: Don't if NONE is specified as input (pythia use case)
193 gutsche 1.35 blockSites = {}
194 slacapra 1.9 if self.datasetPath:
195 gutsche 1.35 blockSites = self.DataDiscoveryAndLocation(cfg_params)
196 ewv 1.131 #DBSDLS-end
197 spiga 1.269 self.conf['blockSites']=blockSites
198    
199 slacapra 1.9 ## Select Splitting
200 spiga 1.269 splitByRun = int(cfg_params.get('CMSSW.split_by_run',0))
201    
202 ewv 1.131 if self.selectNoInput:
203 spiga 1.187 if self.pset == None:
204 ewv 1.276 self.algo = 'ForScript'
205 spiga 1.42 else:
206 spiga 1.271 self.algo = 'NoInput'
207 ewv 1.276 self.conf['managedGenerators']=self.managedGenerators
208     self.conf['generator']=self.generator
209     elif splitByRun ==1:
210     self.algo = 'RunBased'
211 spiga 1.269 else:
212 ewv 1.276 self.algo = 'EventBased'
213    
214     # self.algo = 'LumiBased'
215     splitter = JobSplitter(self.cfg_params,self.conf)
216 spiga 1.269 self.dict = splitter.Algos()[self.algo]()
217 gutsche 1.5
218 spiga 1.293 self.filepath= '%s/arguments.xml'%common.work_space.shareDir()
219     self.rootname= 'arguments'
220 spiga 1.208 # modify Pset only the first time
221     if isNew:
222 spiga 1.293 # modify Pset only the first time
223     if self.pset != None: self.ModifyPset()
224    
225 ewv 1.226 ## Prepare inputSandbox TarBall (only the first time)
226 spiga 1.293 # self.tgzNameWithPath = self.getTarBall(self.executable)
227    
228     def ModifyPset(self):
229     import PsetManipulator as pp
230     PsetEdit = pp.PsetManipulator(self.pset)
231     try:
232     # Add FrameworkJobReport to parameter-set, set max events.
233     # Reset later for data jobs by writeCFG which does all modifications
234 ewv 1.295 PsetEdit.maxEvent(1)
235 spiga 1.293 PsetEdit.skipEvent(0)
236     PsetEdit.psetWriter(self.configFilename())
237     ## If present, add TFileService to output files
238     if not int(self.cfg_params.get('CMSSW.skip_TFileService_output',0)):
239     tfsOutput = PsetEdit.getTFileService()
240     if tfsOutput:
241     if tfsOutput in self.output_file:
242     common.logger.debug(5,"Output from TFileService "+tfsOutput+" already in output files")
243     else:
244     outfileflag = True #output found
245     self.output_file.append(tfsOutput)
246     common.logger.message("Adding "+tfsOutput+" (from TFileService) to list of output files")
247     pass
248     pass
249     ## If present and requested, add PoolOutputModule to output files
250     if int(self.cfg_params.get('CMSSW.get_edm_output',0)):
251     edmOutput = PsetEdit.getPoolOutputModule()
252     if edmOutput:
253     if edmOutput in self.output_file:
254     common.logger.debug(5,"Output from PoolOutputModule "+edmOutput+" already in output files")
255     else:
256     self.output_file.append(edmOutput)
257     common.logger.message("Adding "+edmOutput+" (from PoolOutputModule) to list of output files")
258     pass
259     pass
260 slacapra 1.297 # not required: check anyhow if present, to avoid accidental T2 overload
261     else:
262     edmOutput = PsetEdit.getPoolOutputModule()
263     if edmOutput and (edmOutput not in self.output_file):
264     msg = "ERROR: a PoolOutputModule is present in your ParameteSet %s \n"%self.pset
265     msg +=" but the file produced ( %s ) is not in the list of output files\n"%edmOutput
266     msg += "WARNING: please remove it or, if you wan to keep it, add the file to output_files or use CMSSW.get_edm_output\n"
267     raise CrabException(msg)
268     pass
269     pass
270     except CrabException, msg:
271     common.logger.message(str(msg))
272     msg='Error while manipulating ParameterSet (see previous message, if any): exiting...'
273 spiga 1.293 raise CrabException(msg)
274    
275 gutsche 1.3
276 slacapra 1.1 def DataDiscoveryAndLocation(self, cfg_params):
277    
278 slacapra 1.86 import DataDiscovery
279     import DataLocation
280 gutsche 1.3 common.logger.debug(10,"CMSSW::DataDiscoveryAndLocation()")
281    
282     datasetPath=self.datasetPath
283    
284 slacapra 1.1 ## Contact the DBS
285 gutsche 1.92 common.logger.message("Contacting Data Discovery Services ...")
286 slacapra 1.1 try:
287 spiga 1.208 self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params,self.skip_blocks)
288 slacapra 1.1 self.pubdata.fetchDBSInfo()
289    
290 slacapra 1.41 except DataDiscovery.NotExistingDatasetError, ex :
291 slacapra 1.1 msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
292     raise CrabException(msg)
293 slacapra 1.41 except DataDiscovery.NoDataTierinProvenanceError, ex :
294 slacapra 1.1 msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
295     raise CrabException(msg)
296 slacapra 1.41 except DataDiscovery.DataDiscoveryError, ex:
297 gutsche 1.66 msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
298 slacapra 1.1 raise CrabException(msg)
299    
300 gutsche 1.35 self.filesbyblock=self.pubdata.getFiles()
301 slacapra 1.270 #print self.filesbyblock
302 spiga 1.269 self.conf['pubdata']=self.pubdata
303 gutsche 1.3
304 slacapra 1.1 ## get max number of events
305 ewv 1.192 self.maxEvents=self.pubdata.getMaxEvents()
306 slacapra 1.1
307     ## Contact the DLS and build a list of sites hosting the fileblocks
308     try:
309 slacapra 1.41 dataloc=DataLocation.DataLocation(self.filesbyblock.keys(),cfg_params)
310 gutsche 1.6 dataloc.fetchDLSInfo()
311 slacapra 1.263
312 slacapra 1.41 except DataLocation.DataLocationError , ex:
313 slacapra 1.1 msg = 'ERROR ***: failed Data Location in DLS \n %s '%ex.getErrorMessage()
314     raise CrabException(msg)
315 ewv 1.131
316 slacapra 1.1
317 slacapra 1.270 unsorted_sites = dataloc.getSites()
318     #print "Unsorted :",unsorted_sites
319     sites = self.filesbyblock.fromkeys(self.filesbyblock,'')
320     for lfn in self.filesbyblock.keys():
321     #print lfn
322     if unsorted_sites.has_key(lfn):
323     #print "Found ",lfn
324     sites[lfn]=unsorted_sites[lfn]
325     else:
326     #print "Not Found ",lfn
327     sites[lfn]=[]
328     #print sites
329    
330     #print "Sorted :",sites
331 slacapra 1.264 if len(sites)==0:
332 spiga 1.267 msg = 'ERROR ***: no location for any of the blocks of this dataset: \n\t %s \n'%datasetPath
333     msg += "\tMaybe the dataset is located only at T1's (or at T0), where analysis jobs are not allowed\n"
334     msg += "\tPlease check DataDiscovery page https://cmsweb.cern.ch/dbs_discovery/\n"
335 slacapra 1.264 raise CrabException(msg)
336    
337 gutsche 1.35 allSites = []
338     listSites = sites.values()
339 slacapra 1.63 for listSite in listSites:
340     for oneSite in listSite:
341 gutsche 1.35 allSites.append(oneSite)
342 slacapra 1.291 [allSites.append(it) for it in allSites if not allSites.count(it)]
343 ewv 1.295
344 gutsche 1.3
345 gutsche 1.92 # screen output
346     common.logger.message("Requested dataset: " + datasetPath + " has " + str(self.maxEvents) + " events in " + str(len(self.filesbyblock.keys())) + " blocks.\n")
347    
348 gutsche 1.35 return sites
349 ewv 1.131
350 spiga 1.42
351 spiga 1.208 def split(self, jobParams,firstJobID):
352 ewv 1.276
353 spiga 1.293 jobParams = self.dict['args']
354 spiga 1.269 njobs = self.dict['njobs']
355     self.jobDestination = self.dict['jobDestination']
356 ewv 1.131
357 slacapra 1.263 if njobs==0:
358     raise CrabException("Ask to split "+str(njobs)+" jobs: aborting")
359    
360 gutsche 1.3 # create the empty structure
361     for i in range(njobs):
362     jobParams.append("")
363 ewv 1.131
364 spiga 1.165 listID=[]
365     listField=[]
366 spiga 1.293 listDictions=[]
367     exist= os.path.exists(self.filepath)
368 spiga 1.208 for id in range(njobs):
369     job = id + int(firstJobID)
370 spiga 1.167 listID.append(job+1)
371 spiga 1.162 job_ToSave ={}
372 spiga 1.169 concString = ' '
373 spiga 1.165 argu=''
374 spiga 1.293 str_argu = str(job+1)
375 spiga 1.208 if len(jobParams[id]):
376 ewv 1.295 argu = {'JobID': job+1}
377 spiga 1.293 for i in range(len(jobParams[id])):
378     argu[self.dict['params'][i]]=jobParams[id][i]
379 ewv 1.295 # just for debug
380 spiga 1.293 str_argu += concString.join(jobParams[id])
381     listDictions.append(argu)
382 spiga 1.298 job_ToSave['arguments']= str(job+1)
383 spiga 1.208 job_ToSave['dlsDestination']= self.jobDestination[id]
384 spiga 1.165 listField.append(job_ToSave)
385 ewv 1.295 msg="Job %s Arguments: %s\n"%(str(job+1),str_argu)
386 spiga 1.293 msg+="\t Destination: %s "%(str(self.jobDestination[id]))
387 spiga 1.165 common.logger.debug(5,msg)
388 spiga 1.293 # write xml
389 ewv 1.295 if len(listDictions):
390 spiga 1.293 if exist==False: self.CreateXML()
391     self.addEntry(listDictions)
392 spiga 1.187 common._db.updateJob_(listID,listField)
393 spiga 1.294 # to be improved.... Daniele
394 spiga 1.293 self.tgzNameWithPath = self.getTarBall(self.executable)
395     return
396 gutsche 1.3
397 spiga 1.293 def CreateXML(self):
398     """
399 ewv 1.295 """
400 spiga 1.293 result = IMProvNode( self.rootname )
401     outfile = file( self.filepath, 'w').write(str(result))
402 ewv 1.295 return
403 spiga 1.293
404     def addEntry(self, listDictions):
405     """
406     _addEntry_
407 ewv 1.295
408 spiga 1.293 add an entry to the xml file
409     """
410     from IMProv.IMProvLoader import loadIMProvFile
411     ## load xml
412     improvDoc = loadIMProvFile(self.filepath)
413     entrname= 'Job'
414     for dictions in listDictions:
415     report = IMProvNode(entrname , None, **dictions)
416     improvDoc.addNode(report)
417     outfile = file( self.filepath, 'w').write(str(improvDoc))
418 gutsche 1.3 return
419 ewv 1.131
420 gutsche 1.3 def numberOfJobs(self):
421 spiga 1.269 return self.dict['njobs']
422 gutsche 1.3
423 slacapra 1.1 def getTarBall(self, exe):
424     """
425     Return the TarBall with lib and exe
426     """
427 slacapra 1.242 self.tgzNameWithPath = common.work_space.pathForTgz()+self.tgz_name
428 slacapra 1.1 if os.path.exists(self.tgzNameWithPath):
429     return self.tgzNameWithPath
430    
431     # Prepare a tar gzipped file with user binaries.
432     self.buildTar_(exe)
433    
434     return string.strip(self.tgzNameWithPath)
435    
436     def buildTar_(self, executable):
437    
438     # First of all declare the user Scram area
439     swArea = self.scram.getSWArea_()
440     swReleaseTop = self.scram.getReleaseTop_()
441 ewv 1.131
442 slacapra 1.1 ## check if working area is release top
443     if swReleaseTop == '' or swArea == swReleaseTop:
444 afanfani 1.172 common.logger.debug(3,"swArea = "+swArea+" swReleaseTop ="+swReleaseTop)
445 slacapra 1.1 return
446    
447 slacapra 1.61 import tarfile
448     try: # create tar ball
449     tar = tarfile.open(self.tgzNameWithPath, "w:gz")
450     ## First find the executable
451 slacapra 1.86 if (self.executable != ''):
452 slacapra 1.61 exeWithPath = self.scram.findFile_(executable)
453     if ( not exeWithPath ):
454     raise CrabException('User executable '+executable+' not found')
455 ewv 1.131
456 slacapra 1.61 ## then check if it's private or not
457     if exeWithPath.find(swReleaseTop) == -1:
458     # the exe is private, so we must ship
459     common.logger.debug(5,"Exe "+exeWithPath+" to be tarred")
460     path = swArea+'/'
461 corvo 1.85 # distinguish case when script is in user project area or given by full path somewhere else
462     if exeWithPath.find(path) >= 0 :
463     exe = string.replace(exeWithPath, path,'')
464 slacapra 1.129 tar.add(path+exe,exe)
465 corvo 1.85 else :
466     tar.add(exeWithPath,os.path.basename(executable))
467 slacapra 1.61 pass
468     else:
469     # the exe is from release, we'll find it on WN
470     pass
471 ewv 1.131
472 slacapra 1.61 ## Now get the libraries: only those in local working area
473 slacapra 1.256 tar.dereference=True
474 slacapra 1.61 libDir = 'lib'
475     lib = swArea+'/' +libDir
476     common.logger.debug(5,"lib "+lib+" to be tarred")
477     if os.path.exists(lib):
478     tar.add(lib,libDir)
479 ewv 1.131
480 slacapra 1.61 ## Now check if module dir is present
481     moduleDir = 'module'
482     module = swArea + '/' + moduleDir
483     if os.path.isdir(module):
484     tar.add(module,moduleDir)
485 slacapra 1.256 tar.dereference=False
486 slacapra 1.61
487     ## Now check if any data dir(s) is present
488 spiga 1.179 self.dataExist = False
489 slacapra 1.212 todo_list = [(i, i) for i in os.listdir(swArea+"/src")]
490 slacapra 1.206 while len(todo_list):
491     entry, name = todo_list.pop()
492 slacapra 1.211 if name.startswith('crab_0_') or name.startswith('.') or name == 'CVS':
493 slacapra 1.206 continue
494 slacapra 1.212 if os.path.isdir(swArea+"/src/"+entry):
495 slacapra 1.206 entryPath = entry + '/'
496 slacapra 1.212 todo_list += [(entryPath + i, i) for i in os.listdir(swArea+"/src/"+entry)]
497 slacapra 1.206 if name == 'data':
498     self.dataExist=True
499     common.logger.debug(5,"data "+entry+" to be tarred")
500 slacapra 1.212 tar.add(swArea+"/src/"+entry,"src/"+entry)
501 slacapra 1.206 pass
502     pass
503 ewv 1.182
504 spiga 1.179 ### CMSSW ParameterSet
505     if not self.pset is None:
506     cfg_file = common.work_space.jobDir()+self.configFilename()
507 ewv 1.182 tar.add(cfg_file,self.configFilename())
508 slacapra 1.61
509 fanzago 1.93
510 fanzago 1.152 ## Add ProdCommon dir to tar
511 slacapra 1.211 prodcommonDir = './'
512     prodcommonPath = os.environ['CRABDIR'] + '/' + 'external/'
513 spiga 1.244 neededStuff = ['ProdCommon/__init__.py','ProdCommon/FwkJobRep', 'ProdCommon/CMSConfigTools', \
514 spiga 1.298 'ProdCommon/Core', 'ProdCommon/MCPayloads', 'IMProv', 'ProdCommon/Storage', \
515     'WMCore/__init__.py','WMCore/Algorithms']
516 slacapra 1.214 for file in neededStuff:
517     tar.add(prodcommonPath+file,prodcommonDir+file)
518 spiga 1.179
519     ##### ML stuff
520     ML_file_list=['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py']
521     path=os.environ['CRABDIR'] + '/python/'
522     for file in ML_file_list:
523     tar.add(path+file,file)
524    
525     ##### Utils
526 spiga 1.238 Utils_file_list=['parseCrabFjr.py','writeCfg.py', 'fillCrabFjr.py','cmscp.py']
527 spiga 1.179 for file in Utils_file_list:
528     tar.add(path+file,file)
529 ewv 1.131
530 ewv 1.182 ##### AdditionalFiles
531 slacapra 1.253 tar.dereference=True
532 spiga 1.179 for file in self.additional_inbox_files:
533     tar.add(file,string.split(file,'/')[-1])
534 slacapra 1.253 tar.dereference=False
535 slacapra 1.263 common.logger.debug(5,"Files in "+self.tgzNameWithPath+" : "+str(tar.getnames()))
536 ewv 1.295
537 spiga 1.293 if (os.path.exists(self.filepath)):
538     tar.add(self.filepath,'arguments.xml')
539 ewv 1.182
540 slacapra 1.61 tar.close()
541 mcinquil 1.241 except IOError, exc:
542     common.logger.write(str(exc))
543 slacapra 1.220 raise CrabException('Could not create tar-ball '+self.tgzNameWithPath)
544 mcinquil 1.241 except tarfile.TarError, exc:
545     common.logger.write(str(exc))
546 slacapra 1.206 raise CrabException('Could not create tar-ball '+self.tgzNameWithPath)
547 gutsche 1.72
548     ## check for tarball size
549     tarballinfo = os.stat(self.tgzNameWithPath)
550     if ( tarballinfo.st_size > self.MaxTarBallSize*1024*1024 ) :
551 spiga 1.238 msg = 'Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' + str(self.MaxTarBallSize) \
552 ewv 1.250 +'MB input sandbox limit \n'
553 spiga 1.238 msg += ' and not supported by the direct GRID submission system.\n'
554     msg += ' Please use the CRAB server mode by setting server_name=<NAME> in section [CRAB] of your crab.cfg.\n'
555     msg += ' For further infos please see https://twiki.cern.ch/twiki/bin/view/CMS/CrabServer#CRABSERVER_for_Users'
556     raise CrabException(msg)
557 gutsche 1.72
558 slacapra 1.61 ## create tar-ball with ML stuff
559 slacapra 1.97
560 spiga 1.165 def wsSetupEnvironment(self, nj=0):
561 slacapra 1.1 """
562     Returns part of a job script which prepares
563     the execution environment for the job 'nj'.
564     """
565 ewv 1.276 # FUTURE: Drop support for .cfg when possible
566 ewv 1.184 if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
567     psetName = 'pset.py'
568     else:
569     psetName = 'pset.cfg'
570 slacapra 1.1 # Prepare JobType-independent part
571 ewv 1.160 txt = '\n#Written by cms_cmssw::wsSetupEnvironment\n'
572 fanzago 1.133 txt += 'echo ">>> setup environment"\n'
573 spiga 1.290 txt += 'if [ $middleware == LCG ] || [ $middleware == CAF ] || [ $middleware == LSF ]; then \n'
574 gutsche 1.3 txt += self.wsSetupCMSLCGEnvironment_()
575 ewv 1.283 txt += 'elif [ $middleware == OSG ]; then\n'
576 gutsche 1.43 txt += ' WORKING_DIR=`/bin/mktemp -d $OSG_WN_TMP/cms_XXXXXXXXXXXX`\n'
577 ewv 1.132 txt += ' if [ ! $? == 0 ] ;then\n'
578 fanzago 1.161 txt += ' echo "ERROR ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
579     txt += ' job_exit_code=10016\n'
580     txt += ' func_exit\n'
581 gutsche 1.3 txt += ' fi\n'
582 fanzago 1.133 txt += ' echo ">>> Created working directory: $WORKING_DIR"\n'
583 gutsche 1.3 txt += '\n'
584     txt += ' echo "Change to working directory: $WORKING_DIR"\n'
585     txt += ' cd $WORKING_DIR\n'
586 fanzago 1.133 txt += ' echo ">>> current directory (WORKING_DIR): $WORKING_DIR"\n'
587 ewv 1.131 txt += self.wsSetupCMSOSGEnvironment_()
588 spiga 1.282 #Setup SGE Environment
589 ewv 1.283 txt += 'elif [ $middleware == SGE ]; then\n'
590 spiga 1.282 txt += self.wsSetupCMSLCGEnvironment_()
591    
592 edelmann 1.289 txt += 'elif [ $middleware == ARC ]; then\n'
593     txt += self.wsSetupCMSLCGEnvironment_()
594    
595 gutsche 1.3 txt += 'fi\n'
596 slacapra 1.1
597     # Prepare JobType-specific part
598     scram = self.scram.commandName()
599     txt += '\n\n'
600 fanzago 1.133 txt += 'echo ">>> specific cmssw setup environment:"\n'
601     txt += 'echo "CMSSW_VERSION = '+self.version+'"\n'
602 slacapra 1.1 txt += scram+' project CMSSW '+self.version+'\n'
603     txt += 'status=$?\n'
604     txt += 'if [ $status != 0 ] ; then\n'
605 fanzago 1.161 txt += ' echo "ERROR ==> CMSSW '+self.version+' not found on `hostname`" \n'
606     txt += ' job_exit_code=10034\n'
607 fanzago 1.163 txt += ' func_exit\n'
608 slacapra 1.1 txt += 'fi \n'
609     txt += 'cd '+self.version+'\n'
610 spiga 1.277 txt += 'SOFTWARE_DIR=`pwd`; export SOFTWARE_DIR\n'
611 fanzago 1.133 txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
612 slacapra 1.1 txt += 'eval `'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME`\n'
613 fanzago 1.180 txt += 'if [ $? != 0 ] ; then\n'
614     txt += ' echo "ERROR ==> Problem with the command: "\n'
615     txt += ' echo "eval \`'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME \` at `hostname`"\n'
616     txt += ' job_exit_code=10034\n'
617     txt += ' func_exit\n'
618     txt += 'fi \n'
619 slacapra 1.1 # Handle the arguments:
620     txt += "\n"
621 gutsche 1.7 txt += "## number of arguments (first argument always jobnumber)\n"
622 slacapra 1.1 txt += "\n"
623 spiga 1.165 txt += "if [ $nargs -lt "+str(self.argsList)+" ]\n"
624 slacapra 1.1 txt += "then\n"
625 fanzago 1.161 txt += " echo 'ERROR ==> Too few arguments' +$nargs+ \n"
626     txt += ' job_exit_code=50113\n'
627     txt += " func_exit\n"
628 slacapra 1.1 txt += "fi\n"
629     txt += "\n"
630    
631     # Prepare job-specific part
632     job = common.job_list[nj]
633 ewv 1.131 if (self.datasetPath):
634 spiga 1.238 self.primaryDataset = self.datasetPath.split("/")[1]
635     DataTier = self.datasetPath.split("/")[2]
636 fanzago 1.93 txt += '\n'
637     txt += 'DatasetPath='+self.datasetPath+'\n'
638    
639 spiga 1.238 txt += 'PrimaryDataset='+self.primaryDataset +'\n'
640     txt += 'DataTier='+DataTier+'\n'
641 fanzago 1.96 txt += 'ApplicationFamily=cmsRun\n'
642 fanzago 1.93
643     else:
644 ewv 1.250 self.primaryDataset = 'null'
645 fanzago 1.93 txt += 'DatasetPath=MCDataTier\n'
646     txt += 'PrimaryDataset=null\n'
647     txt += 'DataTier=null\n'
648     txt += 'ApplicationFamily=MCDataTier\n'
649 ewv 1.170 if self.pset != None:
650 spiga 1.42 pset = os.path.basename(job.configFilename())
651     txt += '\n'
652 spiga 1.95 txt += 'cp $RUNTIME_AREA/'+pset+' .\n'
653 spiga 1.296
654 ewv 1.295 txt += 'PreserveSeeds=' + ','.join(self.preserveSeeds) + '; export PreserveSeeds\n'
655     txt += 'IncrementSeeds=' + ','.join(self.incrementSeeds) + '; export IncrementSeeds\n'
656     txt += 'echo "PreserveSeeds: <$PreserveSeeds>"\n'
657     txt += 'echo "IncrementSeeds:<$IncrementSeeds>"\n'
658 slacapra 1.90
659 ewv 1.184 txt += 'mv -f ' + pset + ' ' + psetName + '\n'
660 slacapra 1.1
661    
662 fanzago 1.163 if self.pset != None:
663 ewv 1.184 # FUTURE: Can simply for 2_1_x and higher
664 spiga 1.42 txt += '\n'
665 fanzago 1.285 if self.debug_wrapper == 1:
666 spiga 1.188 txt += 'echo "***** cat ' + psetName + ' *********"\n'
667     txt += 'cat ' + psetName + '\n'
668     txt += 'echo "****** end ' + psetName + ' ********"\n'
669     txt += '\n'
670 fanzago 1.286 txt += 'echo "***********************" \n'
671     txt += 'which edmConfigHash \n'
672     txt += 'echo "***********************" \n'
673 spiga 1.287 if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
674 fanzago 1.286 txt += 'edmConfigHash ' + psetName + ' \n'
675 spiga 1.287 txt += 'PSETHASH=`edmConfigHash ' + psetName + '` \n'
676     else:
677 ewv 1.226 txt += 'PSETHASH=`edmConfigHash < ' + psetName + '` \n'
678 fanzago 1.94 txt += 'echo "PSETHASH = $PSETHASH" \n'
679 fanzago 1.292 #### FEDE temporary fix for noEdm files #####
680     txt += 'if [ -z "$PSETHASH" ]; then \n'
681     txt += ' export PSETHASH=null\n'
682     txt += 'fi \n'
683     #############################################
684 fanzago 1.93 txt += '\n'
685 gutsche 1.3 return txt
686 slacapra 1.176
687 fanzago 1.166 def wsUntarSoftware(self, nj=0):
688 gutsche 1.3 """
689     Put in the script the commands to build an executable
690     or a library.
691     """
692    
693 fanzago 1.166 txt = '\n#Written by cms_cmssw::wsUntarSoftware\n'
694 gutsche 1.3
695     if os.path.isfile(self.tgzNameWithPath):
696 fanzago 1.133 txt += 'echo ">>> tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+' :" \n'
697 slacapra 1.255 txt += 'tar xzf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
698 fanzago 1.285 if self.debug_wrapper==1 :
699 slacapra 1.255 txt += 'tar tzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
700 spiga 1.199 txt += 'ls -Al \n'
701 gutsche 1.3 txt += 'untar_status=$? \n'
702     txt += 'if [ $untar_status -ne 0 ]; then \n'
703 fanzago 1.161 txt += ' echo "ERROR ==> Untarring .tgz file failed"\n'
704     txt += ' job_exit_code=$untar_status\n'
705     txt += ' func_exit\n'
706 gutsche 1.3 txt += 'else \n'
707     txt += ' echo "Successful untar" \n'
708     txt += 'fi \n'
709 gutsche 1.50 txt += '\n'
710 slacapra 1.211 txt += 'echo ">>> Include $RUNTIME_AREA in PYTHONPATH:"\n'
711 gutsche 1.50 txt += 'if [ -z "$PYTHONPATH" ]; then\n'
712 slacapra 1.211 txt += ' export PYTHONPATH=$RUNTIME_AREA/\n'
713 gutsche 1.50 txt += 'else\n'
714 slacapra 1.211 txt += ' export PYTHONPATH=$RUNTIME_AREA/:${PYTHONPATH}\n'
715 fanzago 1.93 txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
716 gutsche 1.50 txt += 'fi\n'
717     txt += '\n'
718    
719 gutsche 1.3 pass
720 ewv 1.131
721 slacapra 1.1 return txt
722 ewv 1.170
723 fanzago 1.166 def wsBuildExe(self, nj=0):
724     """
725     Put in the script the commands to build an executable
726     or a library.
727     """
728    
729     txt = '\n#Written by cms_cmssw::wsBuildExe\n'
730     txt += 'echo ">>> moving CMSSW software directories in `pwd`" \n'
731    
732 ewv 1.170 txt += 'rm -r lib/ module/ \n'
733     txt += 'mv $RUNTIME_AREA/lib/ . \n'
734     txt += 'mv $RUNTIME_AREA/module/ . \n'
735 spiga 1.186 if self.dataExist == True:
736     txt += 'rm -r src/ \n'
737     txt += 'mv $RUNTIME_AREA/src/ . \n'
738 ewv 1.182 if len(self.additional_inbox_files)>0:
739 spiga 1.179 for file in self.additional_inbox_files:
740 spiga 1.191 txt += 'mv $RUNTIME_AREA/'+os.path.basename(file)+' . \n'
741 slacapra 1.214 # txt += 'mv $RUNTIME_AREA/ProdCommon/ . \n'
742     # txt += 'mv $RUNTIME_AREA/IMProv/ . \n'
743 ewv 1.170
744 slacapra 1.211 txt += 'echo ">>> Include $RUNTIME_AREA in PYTHONPATH:"\n'
745 fanzago 1.166 txt += 'if [ -z "$PYTHONPATH" ]; then\n'
746 slacapra 1.211 txt += ' export PYTHONPATH=$RUNTIME_AREA/\n'
747 fanzago 1.166 txt += 'else\n'
748 slacapra 1.211 txt += ' export PYTHONPATH=$RUNTIME_AREA/:${PYTHONPATH}\n'
749 fanzago 1.166 txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
750     txt += 'fi\n'
751     txt += '\n'
752    
753     return txt
754 slacapra 1.1
755 ewv 1.131
756 slacapra 1.1 def executableName(self):
757 ewv 1.192 if self.scriptExe:
758 spiga 1.42 return "sh "
759     else:
760     return self.executable
761 slacapra 1.1
762     def executableArgs(self):
763 ewv 1.160 # FUTURE: This function tests the CMSSW version. Can be simplified as we drop support for old versions
764 ewv 1.276 if self.scriptExe:
765     return self.scriptExe + " $NJob"
766 fanzago 1.115 else:
767 ewv 1.160 ex_args = ""
768 ewv 1.276 ex_args += " -j $RUNTIME_AREA/crab_fjr_$NJob.xml"
769     # Type of config file depends on CMSSW version
770 ewv 1.184 if self.CMSSW_major >= 2 :
771 ewv 1.171 ex_args += " -p pset.py"
772 fanzago 1.115 else:
773 ewv 1.160 ex_args += " -p pset.cfg"
774     return ex_args
775 slacapra 1.1
776     def inputSandbox(self, nj):
777     """
778     Returns a list of filenames to be put in JDL input sandbox.
779     """
780     inp_box = []
781     if os.path.isfile(self.tgzNameWithPath):
782     inp_box.append(self.tgzNameWithPath)
783 spiga 1.243 inp_box.append(common.work_space.jobDir() + self.scriptName)
784 slacapra 1.1 return inp_box
785    
786     def outputSandbox(self, nj):
787     """
788     Returns a list of filenames to be put in JDL output sandbox.
789     """
790     out_box = []
791    
792     ## User Declared output files
793 slacapra 1.54 for out in (self.output_file+self.output_file_sandbox):
794 ewv 1.131 n_out = nj + 1
795 slacapra 1.207 out_box.append(numberFile(out,str(n_out)))
796 slacapra 1.1 return out_box
797    
798    
799     def wsRenameOutput(self, nj):
800     """
801     Returns part of a job script which renames the produced files.
802     """
803    
804 ewv 1.160 txt = '\n#Written by cms_cmssw::wsRenameOutput\n'
805 fanzago 1.148 txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
806     txt += 'echo ">>> current directory content:"\n'
807 fanzago 1.285 if self.debug_wrapper==1:
808 spiga 1.199 txt += 'ls -Al\n'
809 fanzago 1.145 txt += '\n'
810 slacapra 1.54
811 fanzago 1.128 for fileWithSuffix in (self.output_file):
812 slacapra 1.207 output_file_num = numberFile(fileWithSuffix, '$NJob')
813 slacapra 1.1 txt += '\n'
814 gutsche 1.7 txt += '# check output file\n'
815 slacapra 1.106 txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
816 ewv 1.147 if (self.copy_data == 1): # For OSG nodes, file is in $WORKING_DIR, should not be moved to $RUNTIME_AREA
817     txt += ' mv '+fileWithSuffix+' '+output_file_num+'\n'
818 spiga 1.209 txt += ' ln -s `pwd`/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
819 ewv 1.147 else:
820     txt += ' mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
821     txt += ' ln -s $RUNTIME_AREA/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
822 slacapra 1.106 txt += 'else\n'
823 fanzago 1.161 txt += ' job_exit_code=60302\n'
824     txt += ' echo "WARNING: Output file '+fileWithSuffix+' not found"\n'
825 ewv 1.156 if common.scheduler.name().upper() == 'CONDOR_G':
826 gutsche 1.7 txt += ' if [ $middleware == OSG ]; then \n'
827     txt += ' echo "prepare dummy output file"\n'
828     txt += ' echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
829     txt += ' fi \n'
830 slacapra 1.1 txt += 'fi\n'
831 slacapra 1.105 file_list = []
832     for fileWithSuffix in (self.output_file):
833 spiga 1.246 file_list.append(numberFile('$SOFTWARE_DIR/'+fileWithSuffix, '$NJob'))
834 ewv 1.131
835 spiga 1.245 txt += 'file_list="'+string.join(file_list,',')+'"\n'
836 fanzago 1.149 txt += '\n'
837 fanzago 1.148 txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
838     txt += 'echo ">>> current directory content:"\n'
839 fanzago 1.285 if self.debug_wrapper==1:
840 spiga 1.199 txt += 'ls -Al\n'
841 fanzago 1.148 txt += '\n'
842 gutsche 1.7 txt += 'cd $RUNTIME_AREA\n'
843 fanzago 1.133 txt += 'echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
844 slacapra 1.1 return txt
845    
846 slacapra 1.63 def getRequirements(self, nj=[]):
847 slacapra 1.1 """
848 ewv 1.131 return job requirements to add to jdl files
849 slacapra 1.1 """
850     req = ''
851 slacapra 1.47 if self.version:
852 slacapra 1.10 req='Member("VO-cms-' + \
853 slacapra 1.47 self.version + \
854 slacapra 1.10 '", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
855 ewv 1.192 if self.executable_arch:
856 gutsche 1.107 req+=' && Member("VO-cms-' + \
857 slacapra 1.105 self.executable_arch + \
858     '", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
859 gutsche 1.35
860     req = req + ' && (other.GlueHostNetworkAdapterOutboundIP)'
861 afanfani 1.229 if ( common.scheduler.name() == "glitecoll" ) or ( common.scheduler.name() == "glite"):
862 afanfani 1.158 req += ' && other.GlueCEStateStatus == "Production" '
863 gutsche 1.35
864 slacapra 1.1 return req
865 gutsche 1.3
866     def configFilename(self):
867     """ return the config filename """
868 ewv 1.182 # FUTURE: Can remove cfg mode for CMSSW >= 2_1_x
869 ewv 1.184 if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
870 ewv 1.182 return self.name()+'.py'
871     else:
872     return self.name()+'.cfg'
873 gutsche 1.3
874     def wsSetupCMSOSGEnvironment_(self):
875     """
876     Returns part of a job script which is prepares
877     the execution environment and which is common for all CMS jobs.
878     """
879 ewv 1.160 txt = '\n#Written by cms_cmssw::wsSetupCMSOSGEnvironment_\n'
880     txt += ' echo ">>> setup CMS OSG environment:"\n'
881 fanzago 1.133 txt += ' echo "set SCRAM ARCH to ' + self.executable_arch + '"\n'
882     txt += ' export SCRAM_ARCH='+self.executable_arch+'\n'
883 fanzago 1.136 txt += ' echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
884 ewv 1.135 txt += ' if [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n'
885 mkirn 1.40 txt += ' # Use $OSG_APP/cmssoft/cms/cmsset_default.sh to setup cms software\n'
886 fanzago 1.133 txt += ' source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
887     txt += ' else\n'
888 fanzago 1.161 txt += ' echo "ERROR ==> $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
889     txt += ' job_exit_code=10020\n'
890     txt += ' func_exit\n'
891 fanzago 1.133 txt += ' fi\n'
892 gutsche 1.3 txt += '\n'
893 fanzago 1.161 txt += ' echo "==> setup cms environment ok"\n'
894 fanzago 1.136 txt += ' echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
895 gutsche 1.3
896     return txt
897 ewv 1.131
898 gutsche 1.3 def wsSetupCMSLCGEnvironment_(self):
899     """
900     Returns part of a job script which is prepares
901     the execution environment and which is common for all CMS jobs.
902     """
903 ewv 1.160 txt = '\n#Written by cms_cmssw::wsSetupCMSLCGEnvironment_\n'
904     txt += ' echo ">>> setup CMS LCG environment:"\n'
905 fanzago 1.133 txt += ' echo "set SCRAM ARCH and BUILD_ARCH to ' + self.executable_arch + ' ###"\n'
906     txt += ' export SCRAM_ARCH='+self.executable_arch+'\n'
907     txt += ' export BUILD_ARCH='+self.executable_arch+'\n'
908     txt += ' if [ ! $VO_CMS_SW_DIR ] ;then\n'
909 fanzago 1.161 txt += ' echo "ERROR ==> CMS software dir not found on WN `hostname`"\n'
910     txt += ' job_exit_code=10031\n'
911     txt += ' func_exit\n'
912 fanzago 1.133 txt += ' else\n'
913     txt += ' echo "Sourcing environment... "\n'
914     txt += ' if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
915 fanzago 1.161 txt += ' echo "ERROR ==> cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
916     txt += ' job_exit_code=10020\n'
917     txt += ' func_exit\n'
918 fanzago 1.133 txt += ' fi\n'
919     txt += ' echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
920     txt += ' source $VO_CMS_SW_DIR/cmsset_default.sh\n'
921     txt += ' result=$?\n'
922     txt += ' if [ $result -ne 0 ]; then\n'
923 fanzago 1.161 txt += ' echo "ERROR ==> problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
924     txt += ' job_exit_code=10032\n'
925     txt += ' func_exit\n'
926 fanzago 1.133 txt += ' fi\n'
927     txt += ' fi\n'
928     txt += ' \n'
929 fanzago 1.161 txt += ' echo "==> setup cms environment ok"\n'
930 gutsche 1.3 return txt
931 gutsche 1.5
932 spiga 1.238 def wsModifyReport(self, nj):
933 fanzago 1.93 """
934 ewv 1.131 insert the part of the script that modifies the FrameworkJob Report
935 fanzago 1.93 """
936 ewv 1.250
937 fanzago 1.281 txt = ''
938 fanzago 1.286 publish_data = int(self.cfg_params.get('USER.publish_data',0))
939 fanzago 1.292 #if (publish_data == 1):
940     if (self.copy_data == 1):
941 fanzago 1.281 txt = '\n#Written by cms_cmssw::wsModifyReport\n'
942 fanzago 1.292 publish_data = int(self.cfg_params.get('USER.publish_data',0))
943 ewv 1.283
944 spiga 1.238
945     txt += 'if [ $StageOutExitStatus -eq 0 ]; then\n'
946 fanzago 1.248 txt += ' FOR_LFN=$LFNBaseName\n'
947 fanzago 1.175 txt += 'else\n'
948     txt += ' FOR_LFN=/copy_problems/ \n'
949     txt += 'fi\n'
950 ewv 1.182
951 fanzago 1.175 txt += 'echo ">>> Modify Job Report:" \n'
952 fanzago 1.217 txt += 'chmod a+x $RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py\n'
953 fanzago 1.175 txt += 'echo "SE = $SE"\n'
954     txt += 'echo "SE_PATH = $SE_PATH"\n'
955     txt += 'echo "FOR_LFN = $FOR_LFN" \n'
956     txt += 'echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n'
957 fanzago 1.281
958    
959     args = 'fjr $RUNTIME_AREA/crab_fjr_$NJob.xml n_job $NJob for_lfn $FOR_LFN PrimaryDataset $PrimaryDataset ApplicationFamily $ApplicationFamily ApplicationName $executable cmssw_version $CMSSW_VERSION psethash $PSETHASH se_name $SE se_path $SE_PATH'
960 fanzago 1.292 if (publish_data == 1):
961     processedDataset = self.cfg_params['USER.publish_data_name']
962     txt += 'ProcessedDataset='+processedDataset+'\n'
963     txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
964     args += ' UserProcessedDataset $USER-$ProcessedDataset-$PSETHASH'
965 fanzago 1.281
966 fanzago 1.247 txt += 'echo "$RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py '+str(args)+'"\n'
967     txt += '$RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py '+str(args)+'\n'
968 fanzago 1.175 txt += 'modifyReport_result=$?\n'
969     txt += 'if [ $modifyReport_result -ne 0 ]; then\n'
970     txt += ' modifyReport_result=70500\n'
971     txt += ' job_exit_code=$modifyReport_result\n'
972     txt += ' echo "ModifyReportResult=$modifyReport_result" | tee -a $RUNTIME_AREA/$repo\n'
973     txt += ' echo "WARNING: Problem with ModifyJobReport"\n'
974     txt += 'else\n'
975     txt += ' mv NewFrameworkJobReport.xml $RUNTIME_AREA/crab_fjr_$NJob.xml\n'
976 spiga 1.103 txt += 'fi\n'
977 fanzago 1.93 return txt
978 ewv 1.283
979 ewv 1.192 def wsParseFJR(self):
980 spiga 1.189 """
981 ewv 1.192 Parse the FrameworkJobReport to obtain useful infos
982 spiga 1.189 """
983     txt = '\n#Written by cms_cmssw::wsParseFJR\n'
984     txt += 'echo ">>> Parse FrameworkJobReport crab_fjr.xml"\n'
985     txt += 'if [ -s $RUNTIME_AREA/crab_fjr_$NJob.xml ]; then\n'
986     txt += ' if [ -s $RUNTIME_AREA/parseCrabFjr.py ]; then\n'
987 spiga 1.197 txt += ' cmd_out=`python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --dashboard $MonitorID,$MonitorJobID '+self.debugWrap+'`\n'
988 fanzago 1.285 if self.debug_wrapper==1 :
989 spiga 1.197 txt += ' echo "Result of parsing the FrameworkJobReport crab_fjr.xml: $cmd_out"\n'
990     txt += ' executable_exit_status=`python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --exitcode`\n'
991 spiga 1.189 txt += ' if [ $executable_exit_status -eq 50115 ];then\n'
992     txt += ' echo ">>> crab_fjr.xml contents: "\n'
993 spiga 1.222 txt += ' cat $RUNTIME_AREA/crab_fjr_$NJob.xml\n'
994 spiga 1.189 txt += ' echo "Wrong FrameworkJobReport --> does not contain useful info. ExitStatus: $executable_exit_status"\n'
995 spiga 1.197 txt += ' elif [ $executable_exit_status -eq -999 ];then\n'
996     txt += ' echo "ExitStatus from FrameworkJobReport not available. not available. Using exit code of executable from command line."\n'
997 spiga 1.189 txt += ' else\n'
998     txt += ' echo "Extracted ExitStatus from FrameworkJobReport parsing output: $executable_exit_status"\n'
999     txt += ' fi\n'
1000     txt += ' else\n'
1001     txt += ' echo "CRAB python script to parse CRAB FrameworkJobReport crab_fjr.xml is not available, using exit code of executable from command line."\n'
1002     txt += ' fi\n'
1003     #### Patch to check input data reading for CMSSW16x Hopefully we-ll remove it asap
1004 spiga 1.232 txt += ' if [ $executable_exit_status -eq 0 ];then\n'
1005 fanzago 1.273 txt += ' echo ">>> Executable succeded $executable_exit_status"\n'
1006 spiga 1.296 ## This cannot more work given the changes on the Job argumentsJob
1007     """
1008 spiga 1.269 if (self.datasetPath and not (self.dataset_pu or self.useParent==1)) :
1009 spiga 1.189 # VERIFY PROCESSED DATA
1010 fanzago 1.273 txt += ' echo ">>> Verify list of processed files:"\n'
1011     txt += ' echo $InputFiles |tr -d \'\\\\\' |tr \',\' \'\\n\'|tr -d \'"\' > input-files.txt\n'
1012     txt += ' python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --lfn > processed-files.txt\n'
1013     txt += ' cat input-files.txt | sort | uniq > tmp.txt\n'
1014     txt += ' mv tmp.txt input-files.txt\n'
1015     txt += ' echo "cat input-files.txt"\n'
1016     txt += ' echo "----------------------"\n'
1017     txt += ' cat input-files.txt\n'
1018     txt += ' cat processed-files.txt | sort | uniq > tmp.txt\n'
1019     txt += ' mv tmp.txt processed-files.txt\n'
1020     txt += ' echo "----------------------"\n'
1021     txt += ' echo "cat processed-files.txt"\n'
1022     txt += ' echo "----------------------"\n'
1023     txt += ' cat processed-files.txt\n'
1024     txt += ' echo "----------------------"\n'
1025 spiga 1.278 txt += ' diff -qbB input-files.txt processed-files.txt\n'
1026 fanzago 1.273 txt += ' fileverify_status=$?\n'
1027     txt += ' if [ $fileverify_status -ne 0 ]; then\n'
1028     txt += ' executable_exit_status=30001\n'
1029     txt += ' echo "ERROR ==> not all input files processed"\n'
1030     txt += ' echo " ==> list of processed files from crab_fjr.xml differs from list in pset.cfg"\n'
1031     txt += ' echo " ==> diff input-files.txt processed-files.txt"\n'
1032     txt += ' fi\n'
1033 spiga 1.296 """
1034 spiga 1.232 txt += ' fi\n'
1035 spiga 1.189 txt += 'else\n'
1036     txt += ' echo "CRAB FrameworkJobReport crab_fjr.xml is not available, using exit code of executable from command line."\n'
1037     txt += 'fi\n'
1038     txt += '\n'
1039 fanzago 1.279 txt += 'if [ $executable_exit_status -ne 0 ] && [ $executable_exit_status -ne 50115 ] && [ $executable_exit_status -ne 50117 ] && [ $executable_exit_status -ne 30001 ];then\n'
1040 fanzago 1.273 txt += ' echo ">>> Executable failed $executable_exit_status"\n'
1041     txt += ' echo "ExeExitCode=$executable_exit_status" | tee -a $RUNTIME_AREA/$repo\n'
1042     txt += ' echo "EXECUTABLE_EXIT_STATUS = $executable_exit_status"\n'
1043     txt += ' job_exit_code=$executable_exit_status\n'
1044     txt += ' func_exit\n'
1045     txt += 'fi\n\n'
1046 spiga 1.189 txt += 'echo "ExeExitCode=$executable_exit_status" | tee -a $RUNTIME_AREA/$repo\n'
1047     txt += 'echo "EXECUTABLE_EXIT_STATUS = $executable_exit_status"\n'
1048     txt += 'job_exit_code=$executable_exit_status\n'
1049    
1050     return txt
1051    
1052 gutsche 1.5 def setParam_(self, param, value):
1053     self._params[param] = value
1054    
1055     def getParams(self):
1056     return self._params
1057 gutsche 1.8
1058 spiga 1.257 def outList(self,list=False):
1059 mcinquil 1.121 """
1060     check the dimension of the output files
1061     """
1062 spiga 1.169 txt = ''
1063     txt += 'echo ">>> list of expected files on output sandbox"\n'
1064 mcinquil 1.121 listOutFiles = []
1065 ewv 1.170 stdout = 'CMSSW_$NJob.stdout'
1066 spiga 1.169 stderr = 'CMSSW_$NJob.stderr'
1067 spiga 1.268 if len(self.output_file) <= 0:
1068     msg ="WARNING: no output files name have been defined!!\n"
1069     msg+="\tno output files will be reported back/staged\n"
1070     common.logger.message(msg)
1071 fanzago 1.148 if (self.return_data == 1):
1072 spiga 1.157 for file in (self.output_file+self.output_file_sandbox):
1073 slacapra 1.207 listOutFiles.append(numberFile(file, '$NJob'))
1074 spiga 1.169 listOutFiles.append(stdout)
1075     listOutFiles.append(stderr)
1076 ewv 1.156 else:
1077 spiga 1.157 for file in (self.output_file_sandbox):
1078 slacapra 1.207 listOutFiles.append(numberFile(file, '$NJob'))
1079 spiga 1.169 listOutFiles.append(stdout)
1080     listOutFiles.append(stderr)
1081 fanzago 1.161 txt += 'echo "output files: '+string.join(listOutFiles,' ')+'"\n'
1082 spiga 1.157 txt += 'filesToCheck="'+string.join(listOutFiles,' ')+'"\n'
1083 spiga 1.169 txt += 'export filesToCheck\n'
1084 ewv 1.276
1085 spiga 1.257 if list : return self.output_file
1086 ewv 1.170 return txt