ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/cms_cmssw.py

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.300 by spiga, Wed May 20 13:51:45 2009 UTC vs.
Revision 1.314 by spiga, Thu Jun 18 17:23:02 2009 UTC

#	Line 1 | Line 1
1		from JobType import JobType
2	–	from crab_logger import Logger
2		from crab_exceptions import *
3		from crab_util import *
4		import common
#	Line 12 | Line 11 | import os, string, glob
11		class Cmssw(JobType):
12		def __init__(self, cfg_params, ncjobs,skip_blocks, isNew):
13		JobType.__init__(self, 'CMSSW')
14	<	common.logger.debug(3,'CMSSW::__init__')
14	>	common.logger.debug('CMSSW::__init__')
15		self.skip_blocks = skip_blocks
16		self.argsList = 1
17
#	Line 24 | Line 23 | class Cmssw(JobType):
23		size = 9.5
24		if server or common.scheduler.name().upper() in ['LSF','CAF']: size = 99999
25		### D.S.
26	<	self.MaxTarBallSize = float(self.cfg_params.get('EDG.maxtarballsize',size))
26	>	self.MaxTarBallSize = float(self.cfg_params.get('GRID.maxtarballsize',size))
27
28		# number of jobs requested to be created, limit obj splitting
29		self.ncjobs = ncjobs
30
32	–	log = common.logger
33	–
31		self.scram = Scram.Scram(cfg_params)
32		self.additional_inbox_files = []
33		self.scriptExe = ''
#	Line 47 | Line 44 | class Cmssw(JobType):
44		self.fjrFileName = 'crab_fjr.xml'
45
46		self.version = self.scram.getSWVersion()
47	<	common.logger.write("CMSSW version is: "+str(self.version))
47	>	common.logger.log(10-1,"CMSSW version is: "+str(self.version))
48	>
49	>	version_array = self.version.split('_')
50	>	self.CMSSW_major = 0
51	>	self.CMSSW_minor = 0
52	>	self.CMSSW_patch = 0
53		try:
54	<	type, self.CMSSW_major, self.CMSSW_minor, self.CMSSW_patch = tuple(self.version.split('_'))
54	>	self.CMSSW_major = int(version_array[1])
55	>	self.CMSSW_minor = int(version_array[2])
56	>	self.CMSSW_patch = int(version_array[3])
57		except:
58		msg = "Cannot parse CMSSW version string: " + self.version + " for major and minor release number!"
59		raise CrabException(msg)
#	Line 71 | Line 75 | class Cmssw(JobType):
75		self.dataset_pu = cfg_params.get('CMSSW.dataset_pu', None)
76
77		tmp =  cfg_params['CMSSW.datasetpath']
78	<	log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
78	>	common.logger.log(10-1, "CMSSW::CMSSW(): datasetPath = "+tmp)
79
80		if tmp =='':
81		msg = "Error: datasetpath not defined "
#	Line 90 | Line 94 | class Cmssw(JobType):
94		if self.debug_wrapper == 1: self.debugWrap='--debug'
95
96		## now the application
97	<	self.managedGenerators = ['madgraph','comphep']
97	>	self.managedGenerators = ['madgraph', 'comphep', 'lhe']
98		self.generator = cfg_params.get('CMSSW.generator','pythia').lower()
99		self.executable = cfg_params.get('CMSSW.executable','cmsRun')
100	<	log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable)
100	>	common.logger.log(10-1, "CMSSW::CMSSW(): executable = "+self.executable)
101
102		if not cfg_params.has_key('CMSSW.pset'):
103		raise CrabException("PSet file missing. Cannot run cmsRun ")
104		self.pset = cfg_params['CMSSW.pset']
105	<	log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset)
105	>	common.logger.log(10-1, "Cmssw::Cmssw(): PSet file = "+self.pset)
106		if self.pset.lower() != 'none' :
107		if (not os.path.exists(self.pset)):
108		raise CrabException("User defined PSet file "+self.pset+" does not exist")
#	Line 130 | Line 134 | class Cmssw(JobType):
134		raise CrabException(msg)
135		self.additional_inbox_files.append(string.strip(self.scriptExe))
136
137	+	self.AdditionalArgs = cfg_params.get('USER.script_arguments',None)
138	+	if self.AdditionalArgs : self.AdditionalArgs = string.replace(self.AdditionalArgs,',',' ')
139	+
140		if self.datasetPath == None and self.pset == None and self.scriptExe == '' :
141		msg ="Error. script_exe  not defined"
142		raise CrabException(msg)
#	Line 158 | Line 165 | class Cmssw(JobType):
165		self.additional_inbox_files.append(string.strip(file))
166		pass
167		pass
168	<	common.logger.debug(5,"Additional input files: "+str(self.additional_inbox_files))
168	>	common.logger.debug("Additional input files: "+str(self.additional_inbox_files))
169		pass
170
171
#	Line 240 | Line 247 | class Cmssw(JobType):
247		tfsOutput = PsetEdit.getTFileService()
248		if tfsOutput:
249		if tfsOutput in self.output_file:
250	<	common.logger.debug(5,"Output from TFileService "+tfsOutput+" already in output files")
250	>	common.logger.debug("Output from TFileService "+tfsOutput+" already in output files")
251		else:
252		outfileflag = True #output found
253		self.output_file.append(tfsOutput)
254	<	common.logger.message("Adding "+tfsOutput+" (from TFileService) to list of output files")
254	>	common.logger.info("Adding "+tfsOutput+" (from TFileService) to list of output files")
255		pass
256		pass
257		## If present and requested, add PoolOutputModule to output files
258	+	edmOutput = PsetEdit.getPoolOutputModule()
259		if int(self.cfg_params.get('CMSSW.get_edm_output',0)):
252	–	edmOutput = PsetEdit.getPoolOutputModule()
260		if edmOutput:
261		if edmOutput in self.output_file:
262	<	common.logger.debug(5,"Output from PoolOutputModule "+edmOutput+" already in output files")
262	>	common.logger.debug("Output from PoolOutputModule "+edmOutput+" already in output files")
263		else:
264		self.output_file.append(edmOutput)
265	<	common.logger.message("Adding "+edmOutput+" (from PoolOutputModule) to list of output files")
265	>	common.logger.info("Adding "+edmOutput+" (from PoolOutputModule) to list of output files")
266		pass
267		pass
268		# not required: check anyhow if present, to avoid accidental T2 overload
269		else:
263	–	edmOutput = PsetEdit.getPoolOutputModule()
270		if edmOutput and (edmOutput not in self.output_file):
271		msg = "ERROR: a PoolOutputModule is present in your ParameteSet %s \n"%self.pset
272		msg +="         but the file produced ( %s ) is not in the list of output files\n"%edmOutput
273		msg += "WARNING: please remove it. If you want to keep it, add the file to output_files or use CMSSW.get_edm_output\n"
274	<	raise CrabException(msg)
274	>	if int(self.cfg_params.get('CMSSW.ignore_edm_output',0)):
275	>	msg +=" CMSSW.ignore_edm_output==True : Hope you know what you are doing...\n"
276	>	common.logger.info(msg)
277	>	else:
278	>	raise CrabException(msg)
279		pass
280		pass
281	+
282	+	if (PsetEdit.getBadFilesSetting()):
283	+	msg = "WARNING: You have set skipBadFiles to True. This will continue processing on some errors and you may not be notified."
284	+	common.logger.info(msg)
285	+
286		except CrabException, msg:
287	<	common.logger.message(str(msg))
287	>	common.logger.info(str(msg))
288		msg='Error while manipulating ParameterSet (see previous message, if any): exiting...'
289		raise CrabException(msg)
290
#	Line 278 | Line 293 | class Cmssw(JobType):
293
294		import DataDiscovery
295		import DataLocation
296	<	common.logger.debug(10,"CMSSW::DataDiscoveryAndLocation()")
296	>	common.logger.log(10-1,"CMSSW::DataDiscoveryAndLocation()")
297
298		datasetPath=self.datasetPath
299
300		## Contact the DBS
301	<	common.logger.message("Contacting Data Discovery Services ...")
301	>	common.logger.info("Contacting Data Discovery Services ...")
302		try:
303		self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params,self.skip_blocks)
304		self.pubdata.fetchDBSInfo()
#	Line 316 | Line 331 | class Cmssw(JobType):
331
332
333		unsorted_sites = dataloc.getSites()
319	–	#print "Unsorted :",unsorted_sites
334		sites = self.filesbyblock.fromkeys(self.filesbyblock,'')
335		for lfn in self.filesbyblock.keys():
322	–	#print lfn
336		if unsorted_sites.has_key(lfn):
324	–	#print "Found ",lfn
337		sites[lfn]=unsorted_sites[lfn]
338		else:
327	–	#print "Not Found ",lfn
339		sites[lfn]=[]
329	–	#print sites
340
331	–	#print "Sorted :",sites
341		if len(sites)==0:
342		msg = 'ERROR ***: no location for any of the blocks of this dataset: \n\t %s \n'%datasetPath
343		msg += "\tMaybe the dataset is located only at T1's (or at T0), where analysis jobs are not allowed\n"
#	Line 344 | Line 353 | class Cmssw(JobType):
353
354
355		# screen output
356	<	common.logger.message("Requested dataset: " + datasetPath + " has " + str(self.maxEvents) + " events in " + str(len(self.filesbyblock.keys())) + " blocks.\n")
356	>	common.logger.info("Requested dataset: " + datasetPath + " has " + str(self.maxEvents) + " events in " + str(len(self.filesbyblock.keys())) + " blocks.\n")
357
358		return sites
359
#	Line 379 | Line 388 | class Cmssw(JobType):
388		argu[self.dict['params'][i]]=jobParams[id][i]
389		# just for debug
390		str_argu += concString.join(jobParams[id])
391	<	listDictions.append(argu)
391	>	if argu != '': listDictions.append(argu)
392		job_ToSave['arguments']= str(job+1)
393		job_ToSave['dlsDestination']= self.jobDestination[id]
394		listField.append(job_ToSave)
395	+	from ProdCommon.SiteDB.CmsSiteMapper import CmsSEMap
396	+	cms_se = CmsSEMap()
397		msg="Job  %s  Arguments:  %s\n"%(str(job+1),str_argu)
398		msg+="\t  Destination: %s "%(str(self.jobDestination[id]))
399	<	common.logger.debug(5,msg)
399	>	SEDestination = [cms_se[dest] for dest in self.jobDestination[id]]
400	>	msg+="\t  CMSDestination: %s "%(str(SEDestination))
401	>	common.logger.log(10-1,msg)
402		# write xml
403		if len(listDictions):
404		if exist==False: self.CreateXML()
#	Line 394 | Line 407 | class Cmssw(JobType):
407		common._db.updateJob_(listID,listField)
408		self.zipTarFile()
409		return
410	<
410	>
411		def addXMLfile(self):
412
413		import tarfile
414	<	# try:
415	<	print self.argsFile
416	<	tar = tarfile.open(self.tarNameWithPath, "a")
417	<	tar.add(self.argsFile, os.path.basename(self.argsFile))
418	<	tar.close()
419	<	## except:
420	<	#     pass
414	>	try:
415	>	tar = tarfile.open(self.tarNameWithPath, "a")
416	>	tar.add(self.argsFile, os.path.basename(self.argsFile))
417	>	tar.close()
418	>	except IOError, exc:
419	>	msg = 'Could not add %s to %s \n'%(self.argsFile,self.tarNameWithPath)
420	>	msg += str(exc)
421	>	raise CrabException(msg)
422	>	except tarfile.TarError, exc:
423	>	msg = 'Could not add %s to %s \n'%(self.argsFile,self.tarNameWithPath)
424	>	msg += str(exc)
425	>	raise CrabException(msg)
426
409	–
427		def CreateXML(self):
428		"""
429		"""
#	Line 454 | Line 471 | class Cmssw(JobType):
471
472		## check if working area is release top
473		if swReleaseTop == '' or swArea == swReleaseTop:
474	<	common.logger.debug(3,"swArea = "+swArea+" swReleaseTop ="+swReleaseTop)
474	>	common.logger.debug("swArea = "+swArea+" swReleaseTop ="+swReleaseTop)
475		return
476
477		import tarfile
#	Line 470 | Line 487 | class Cmssw(JobType):
487		## then check if it's private or not
488		if exeWithPath.find(swReleaseTop) == -1:
489		# the exe is private, so we must ship
490	<	common.logger.debug(5,"Exe "+exeWithPath+" to be tarred")
490	>	common.logger.debug("Exe "+exeWithPath+" to be tarred")
491		path = swArea+'/'
492		# distinguish case when script is in user project area or given by full path somewhere else
493		if exeWithPath.find(path) >= 0 :
#	Line 487 | Line 504 | class Cmssw(JobType):
504		tar.dereference=True
505		libDir = 'lib'
506		lib = swArea+'/' +libDir
507	<	common.logger.debug(5,"lib "+lib+" to be tarred")
507	>	common.logger.debug("lib "+lib+" to be tarred")
508		if os.path.exists(lib):
509		tar.add(lib,libDir)
510
#	Line 510 | Line 527 | class Cmssw(JobType):
527		todo_list += [(entryPath + i, i) for i in  os.listdir(swArea+"/src/"+entry)]
528		if name == 'data':
529		self.dataExist=True
530	<	common.logger.debug(5,"data "+entry+" to be tarred")
530	>	common.logger.debug("data "+entry+" to be tarred")
531		tar.add(swArea+"/src/"+entry,"src/"+entry)
532		pass
533		pass
#	Line 520 | Line 537 | class Cmssw(JobType):
537		cfg_file = common.work_space.jobDir()+self.configFilename()
538		tar.add(cfg_file,self.configFilename())
539
540	+	try:
541	+	crab_cfg_file = common.work_space.shareDir()+'/crab.cfg'
542	+	tar.add(crab_cfg_file,'crab.cfg')
543	+	except:
544	+	pass
545
546		## Add ProdCommon dir to tar
547		prodcommonDir = './'
#	Line 546 | Line 568 | class Cmssw(JobType):
568		for file in self.additional_inbox_files:
569		tar.add(file,string.split(file,'/')[-1])
570		tar.dereference=False
571	<	common.logger.debug(5,"Files in "+self.tarNameWithPath+" : "+str(tar.getnames()))
571	>	common.logger.log(10-1,"Files in "+self.tarNameWithPath+" : "+str(tar.getnames()))
572
573		tar.close()
574		except IOError, exc:
575	<	common.logger.write(str(exc))
576	<	raise CrabException('Could not create tar-ball '+self.tarNameWithPath)
575	>	msg = 'Could not create tar-ball %s \n'%self.tarNameWithPath
576	>	msg += str(exc)
577	>	raise CrabException(msg)
578		except tarfile.TarError, exc:
579	<	common.logger.write(str(exc))
580	<	raise CrabException('Could not create tar-ball '+self.tarNameWithPath)
581	<
582	<	def zipTarFile(self):
579	>	msg = 'Could not create tar-ball %s \n'%self.tarNameWithPath
580	>	msg += str(exc)
581	>	raise CrabException(msg)
582	>
583	>	def zipTarFile(self):
584
585	<	cmd = "gzip -c %s > %s "%(self.tarNameWithPath,self.tgzNameWithPath)
585	>	cmd = "gzip -c %s > %s "%(self.tarNameWithPath,self.tgzNameWithPath)
586		res=runCommand(cmd)
587
588		tarballinfo = os.stat(self.tgzNameWithPath)
#	Line 672 | Line 696 | class Cmssw(JobType):
696		txt += 'echo "IncrementSeeds:<$IncrementSeeds>"\n'
697
698		txt += 'mv -f ' + pset + ' ' + psetName + '\n'
699	<
676	<
677	<	if self.pset != None:
678	<	# FUTURE: Can simply for 2_1_x and higher
679	<	txt += '\n'
680	<	if self.debug_wrapper == 1:
681	<	txt += 'echo "***** cat ' + psetName + ' *********"\n'
682	<	txt += 'cat ' + psetName + '\n'
683	<	txt += 'echo "****** end ' + psetName + ' ********"\n'
684	<	txt += '\n'
685	<	txt += 'echo "***********************" \n'
686	<	txt += 'which edmConfigHash \n'
687	<	txt += 'echo "***********************" \n'
688	<	if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
689	<	txt += 'edmConfigHash ' + psetName + ' \n'
690	<	txt += 'PSETHASH=`edmConfigHash ' + psetName + '` \n'
691	<	else:
692	<	txt += 'PSETHASH=`edmConfigHash < ' + psetName + '` \n'
693	<	txt += 'echo "PSETHASH = $PSETHASH" \n'
694	<	#### FEDE temporary fix for noEdm files #####
695	<	txt += 'if [ -z "$PSETHASH" ]; then \n'
696	<	txt += '   export PSETHASH=null\n'
697	<	txt += 'fi \n'
698	<	#############################################
699	>	else:
700		txt += '\n'
701	+	txt += 'export AdditionalArgs=%s\n'%(self.AdditionalArgs)
702	+
703		return txt
704
705		def wsUntarSoftware(self, nj=0):
#	Line 764 | Line 767 | class Cmssw(JobType):
767		txt += 'fi\n'
768		txt += '\n'
769
770	+	if self.pset != None:
771	+	# FUTURE: Drop support for .cfg when possible
772	+	if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
773	+	psetName = 'pset.py'
774	+	else:
775	+	psetName = 'pset.cfg'
776	+	# FUTURE: Can simply for 2_1_x and higher
777	+	txt += '\n'
778	+	if self.debug_wrapper == 1:
779	+	txt += 'echo "***** cat ' + psetName + ' *********"\n'
780	+	txt += 'cat ' + psetName + '\n'
781	+	txt += 'echo "****** end ' + psetName + ' ********"\n'
782	+	txt += '\n'
783	+	txt += 'echo "***********************" \n'
784	+	txt += 'which edmConfigHash \n'
785	+	txt += 'echo "***********************" \n'
786	+	if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
787	+	txt += 'edmConfigHash ' + psetName + ' \n'
788	+	txt += 'PSETHASH=`edmConfigHash ' + psetName + '` \n'
789	+	else:
790	+	txt += 'PSETHASH=`edmConfigHash < ' + psetName + '` \n'
791	+	txt += 'echo "PSETHASH = $PSETHASH" \n'
792	+	#### FEDE temporary fix for noEdm files #####
793	+	txt += 'if [ -z "$PSETHASH" ]; then \n'
794	+	txt += '   export PSETHASH=null\n'
795	+	txt += 'fi \n'
796	+	#############################################
797	+	txt += '\n'
798		return txt
799
800
#	Line 776 | Line 807 | class Cmssw(JobType):
807		def executableArgs(self):
808		# FUTURE: This function tests the CMSSW version. Can be simplified as we drop support for old versions
809		if self.scriptExe:
810	<	return self.scriptExe + " $NJob"
810	>	return self.scriptExe + " $NJob $AdditionalArgs"
811		else:
812		ex_args = ""
813		ex_args += " -j $RUNTIME_AREA/crab_fjr_$NJob.xml"
#	Line 1017 | Line 1048 | class Cmssw(JobType):
1048		#### Patch to check input data reading for CMSSW16x Hopefully we-ll remove it asap
1049		txt += '    if [ $executable_exit_status -eq 0 ];then\n'
1050		txt += '        echo ">>> Executable succeded  $executable_exit_status"\n'
1051	<	## This cannot more work given the changes on the Job argumentsJob
1051	>	## This cannot more work given the changes on the Job argumentsJob
1052		"""
1053		if (self.datasetPath and not (self.dataset_pu or self.useParent==1)) :
1054		# VERIFY PROCESSED DATA
#	Line 1081 | Line 1112 | class Cmssw(JobType):
1112		if len(self.output_file) <= 0:
1113		msg ="WARNING: no output files name have been defined!!\n"
1114		msg+="\tno output files will be reported back/staged\n"
1115	<	common.logger.message(msg)
1115	>	common.logger.info(msg)
1116		if (self.return_data == 1):
1117		for file in (self.output_file+self.output_file_sandbox):
1118		listOutFiles.append(numberFile(file, '$NJob'))

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines