ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/cms_cmssw.py

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.75 by gutsche, Sun Apr 8 23:50:33 2007 UTC vs.
Revision 1.128 by fanzago, Thu Oct 11 16:23:44 2007 UTC

#	Line 2 | Line 2 | from JobType import JobType
2		from crab_logger import Logger
3		from crab_exceptions import *
4		from crab_util import *
5	+	from BlackWhiteListParser import BlackWhiteListParser
6		import common
6	–	import PsetManipulator
7	–	import DataDiscovery
8	–	import DataDiscovery_DBS2
9	–	import DataLocation
7		import Scram
8
9	<	import os, string, re, shutil, glob
9	>	import os, string, glob
10
11		class Cmssw(JobType):
12		def __init__(self, cfg_params, ncjobs):
13		JobType.__init__(self, 'CMSSW')
14		common.logger.debug(3,'CMSSW::__init__')
15
19	–	# Marco.
16		self._params = {}
17		self.cfg_params = cfg_params
18
19	+	# init BlackWhiteListParser
20	+	self.blackWhiteListParser = BlackWhiteListParser(cfg_params)
21	+
22		try:
23		self.MaxTarBallSize = float(self.cfg_params['EDG.maxtarballsize'])
24		except KeyError:
25	<	self.MaxTarBallSize = 100.0
25	>	self.MaxTarBallSize = 9.5
26
27		# number of jobs requested to be created, limit obj splitting
28		self.ncjobs = ncjobs
#	Line 36 | Line 35 | class Cmssw(JobType):
35		self.executable = ''
36		self.executable_arch = self.scram.getArch()
37		self.tgz_name = 'default.tgz'
38	+	self.additional_tgz_name = 'additional.tgz'
39		self.scriptName = 'CMSSW.sh'
40		self.pset = ''      #scrip use case Da
41		self.datasetPath = '' #scrip use case Da
#	Line 44 | Line 44 | class Cmssw(JobType):
44		self.fjrFileName = 'crab_fjr.xml'
45
46		self.version = self.scram.getSWVersion()
47	+
48	+	#
49	+	# Try to block creation in case of arch/version mismatch
50	+	#
51	+
52	+	a = string.split(self.version, "_")
53	+
54	+	if int(a[1]) == 1 and (int(a[2]) < 5 and self.executable_arch.find('slc4') == 0):
55	+	msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch)
56	+	raise CrabException(msg)
57	+	if int(a[1]) == 1 and (int(a[2]) >= 5 and self.executable_arch.find('slc3') == 0):
58	+	msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch)
59	+	raise CrabException(msg)
60	+
61		common.taskDB.setDict('codeVersion',self.version)
62		self.setParam_('application', self.version)
63
#	Line 51 | Line 65 | class Cmssw(JobType):
65
66		## get DBS mode
67		try:
68	<	self.use_dbs_2 = int(self.cfg_params['CMSSW.use_dbs_2'])
68	>	self.use_dbs_1 = int(self.cfg_params['CMSSW.use_dbs_1'])
69		except KeyError:
70	<	self.use_dbs_2 = 0
70	>	self.use_dbs_1 = 0
71
72		try:
73		tmp =  cfg_params['CMSSW.datasetpath']
#	Line 74 | Line 88 | class Cmssw(JobType):
88		self.setParam_('dataset', 'None')
89		self.setParam_('owner', 'None')
90		else:
91	<	datasetpath_split = self.datasetPath.split("/")
92	<	if self.use_dbs_2 == 1 :
93	<	self.setParam_('dataset', datasetpath_split[1])
94	<	self.setParam_('owner', datasetpath_split[2])
95	<	else :
96	<	self.setParam_('dataset', datasetpath_split[1])
97	<	self.setParam_('owner', datasetpath_split[-1])
98	<
91	>	try:
92	>	datasetpath_split = self.datasetPath.split("/")
93	>	# standard style
94	>	self.setParam_('datasetFull', self.datasetPath)
95	>	if self.use_dbs_1 == 1 :
96	>	self.setParam_('dataset', datasetpath_split[1])
97	>	self.setParam_('owner', datasetpath_split[-1])
98	>	else:
99	>	self.setParam_('dataset', datasetpath_split[1])
100	>	self.setParam_('owner', datasetpath_split[2])
101	>	except:
102	>	self.setParam_('dataset', self.datasetPath)
103	>	self.setParam_('owner', self.datasetPath)
104	>
105		self.setTaskid_()
106		self.setParam_('taskId', self.cfg_params['taskId'])
107
#	Line 131 | Line 151 | class Cmssw(JobType):
151		self.output_file.append(tmp)
152		pass
153		else:
154	<	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available")
154	>	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
155		pass
156		pass
157		except KeyError:
158	<	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available")
158	>	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
159		pass
160
161		# script_exe file as additional file in inputSandbox
#	Line 161 | Line 181 | class Cmssw(JobType):
181		tmp = string.strip(tmp)
182		dirname = ''
183		if not tmp[0]=="/": dirname = "."
184	<	files = glob.glob(os.path.join(dirname, tmp))
184	>	files = []
185	>	if string.find(tmp,"*")>-1:
186	>	files = glob.glob(os.path.join(dirname, tmp))
187	>	if len(files)==0:
188	>	raise CrabException("No additional input file found with this pattern: "+tmp)
189	>	else:
190	>	files.append(tmp)
191		for file in files:
192		if not os.path.exists(file):
193		raise CrabException("Additional input file not found: "+file)
194		pass
195	<	storedFile = common.work_space.shareDir()+file
196	<	shutil.copyfile(file, storedFile)
197	<	self.additional_inbox_files.append(string.strip(storedFile))
195	>	# fname = string.split(file, '/')[-1]
196	>	# storedFile = common.work_space.pathForTgz()+'share/'+fname
197	>	# shutil.copyfile(file, storedFile)
198	>	self.additional_inbox_files.append(string.strip(file))
199		pass
200		pass
201		common.logger.debug(5,"Additional input files: "+str(self.additional_inbox_files))
#	Line 226 | Line 253 | class Cmssw(JobType):
253		except KeyError:
254		self.sourceSeedVtx = None
255		common.logger.debug(5,"No vertex seed given")
256	+
257	+	try:
258	+	self.sourceSeedG4 = int(cfg_params['CMSSW.g4_seed'])
259	+	except KeyError:
260	+	self.sourceSeedG4 = None
261	+	common.logger.debug(5,"No g4 sim hits seed given")
262	+
263	+	try:
264	+	self.sourceSeedMix = int(cfg_params['CMSSW.mix_seed'])
265	+	except KeyError:
266	+	self.sourceSeedMix = None
267	+	common.logger.debug(5,"No mix seed given")
268	+
269		try:
270		self.firstRun = int(cfg_params['CMSSW.first_run'])
271		except KeyError:
272		self.firstRun = None
273		common.logger.debug(5,"No first run given")
274		if self.pset != None: #CarlosDaniele
275	<	self.PsetEdit = PsetManipulator.PsetManipulator(self.pset) #Daniele Pset
275	>	ver = string.split(self.version,"_")
276	>	if (int(ver[1])>=1 and int(ver[2])>=5):
277	>	import PsetManipulator150 as pp
278	>	else:
279	>	import PsetManipulator as pp
280	>	PsetEdit = pp.PsetManipulator(self.pset) #Daniele Pset
281
282		#DBSDLS-start
283		## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
#	Line 254 | Line 299 | class Cmssw(JobType):
299		self.jobSplittingForScript()
300		else:
301		self.jobSplittingNoInput()
302	<	else:
302	>	else:
303		self.jobSplittingByBlocks(blockSites)
304
305		# modify Pset
#	Line 262 | Line 307 | class Cmssw(JobType):
307		try:
308		if (self.datasetPath): # standard job
309		# allow to processa a fraction of events in a file
310	<	self.PsetEdit.inputModule("INPUT")
311	<	self.PsetEdit.maxEvent("INPUTMAXEVENTS")
312	<	self.PsetEdit.skipEvent("INPUTSKIPEVENTS")
310	>	PsetEdit.inputModule("INPUT")
311	>	PsetEdit.maxEvent("INPUTMAXEVENTS")
312	>	PsetEdit.skipEvent("INPUTSKIPEVENTS")
313		else:  # pythia like job
314	<	self.PsetEdit.maxEvent(self.eventsPerJob)
314	>	PsetEdit.maxEvent(self.eventsPerJob)
315		if (self.firstRun):
316	<	self.PsetEdit.pythiaFirstRun("INPUTFIRSTRUN")  #First Run
316	>	PsetEdit.pythiaFirstRun("INPUTFIRSTRUN")  #First Run
317		if (self.sourceSeed) :
318	<	self.PsetEdit.pythiaSeed("INPUT")
318	>	PsetEdit.pythiaSeed("INPUT")
319		if (self.sourceSeedVtx) :
320	<	self.PsetEdit.pythiaSeedVtx("INPUTVTX")
320	>	PsetEdit.vtxSeed("INPUTVTX")
321	>	if (self.sourceSeedG4) :
322	>	PsetEdit.g4Seed("INPUTG4")
323	>	if (self.sourceSeedMix) :
324	>	PsetEdit.mixSeed("INPUTMIX")
325		# add FrameworkJobReport to parameter-set
326	<	self.PsetEdit.addCrabFJR(self.fjrFileName)
327	<	self.PsetEdit.psetWriter(self.configFilename())
326	>	PsetEdit.addCrabFJR(self.fjrFileName)
327	>	PsetEdit.psetWriter(self.configFilename())
328		except:
329		msg='Error while manipuliating ParameterSet: exiting...'
330		raise CrabException(msg)
331
332		def DataDiscoveryAndLocation(self, cfg_params):
333
334	+	import DataDiscovery
335	+	import DataDiscovery_DBS2
336	+	import DataLocation
337		common.logger.debug(10,"CMSSW::DataDiscoveryAndLocation()")
338
339		datasetPath=self.datasetPath
340
341		## Contact the DBS
342	<	common.logger.message("Contacting DBS...")
342	>	common.logger.message("Contacting Data Discovery Services ...")
343		try:
344
345	<	if self.use_dbs_2 == 1 :
294	<	self.pubdata=DataDiscovery_DBS2.DataDiscovery_DBS2(datasetPath, cfg_params)
295	<	else :
345	>	if self.use_dbs_1 == 1 :
346		self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params)
347	+	else :
348	+	self.pubdata=DataDiscovery_DBS2.DataDiscovery_DBS2(datasetPath, cfg_params)
349		self.pubdata.fetchDBSInfo()
350
351		except DataDiscovery.NotExistingDatasetError, ex :
#	Line 315 | Line 367 | class Cmssw(JobType):
367		msg = 'ERROR ***: failed Data Discovery in DBS :  %s'%ex.getErrorMessage()
368		raise CrabException(msg)
369
318	–	## get list of all required data in the form of dbs paths  (dbs path = /dataset/datatier/owner)
319	–	common.logger.message("Required data are :"+self.datasetPath)
320	–
370		self.filesbyblock=self.pubdata.getFiles()
371		self.eventsbyblock=self.pubdata.getEventsPerBlock()
372		self.eventsbyfile=self.pubdata.getEventsPerFile()
373
374		## get max number of events
375		self.maxEvents=self.pubdata.getMaxEvents() ##  self.maxEvents used in Creator.py
327	–	common.logger.message("The number of available events is %s\n"%self.maxEvents)
376
329	–	common.logger.message("Contacting DLS...")
377		## Contact the DLS and build a list of sites hosting the fileblocks
378		try:
379		dataloc=DataLocation.DataLocation(self.filesbyblock.keys(),cfg_params)
#	Line 344 | Line 391 | class Cmssw(JobType):
391		allSites.append(oneSite)
392		allSites = self.uniquelist(allSites)
393
394	<	common.logger.message("Sites ("+str(len(allSites))+") hosting part/all of dataset: "+str(allSites))
395	<	common.logger.debug(6, "List of Sites: "+str(allSites))
394	>	# screen output
395	>	common.logger.message("Requested dataset: " + datasetPath + " has " + str(self.maxEvents) + " events in " + str(len(self.filesbyblock.keys())) + " blocks.\n")
396	>
397		return sites
398
399		def jobSplittingByBlocks(self, blockSites):
#	Line 407 | Line 455 | class Cmssw(JobType):
455		jobCount = 0
456		list_of_lists = []
457
458	+	# list tracking which jobs are in which jobs belong to which block
459	+	jobsOfBlock = {}
460	+
461		# ---- Iterate over the blocks in the dataset until ---- #
462		# ---- we've met the requested total # of events    ---- #
463		while ( (eventsRemaining > 0) and (blockCount < numBlocksInDataset) and (jobCount < totalNumberOfJobs)):
464		block = blocks[blockCount]
465		blockCount += 1
466	+	if block not in jobsOfBlock.keys() :
467	+	jobsOfBlock[block] = []
468
469		if self.eventsbyblock.has_key(block) :
470		numEventsInBlock = self.eventsbyblock[block]
#	Line 461 | Line 514 | class Cmssw(JobType):
514		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(filesEventCount - jobSkipEventCount)+" events (last file in block).")
515		self.jobDestination.append(blockSites[block])
516		common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
517	+	# fill jobs of block dictionary
518	+	jobsOfBlock[block].append(jobCount+1)
519		# reset counter
520		jobCount = jobCount + 1
521		totalEventCount = totalEventCount + filesEventCount - jobSkipEventCount
#	Line 484 | Line 539 | class Cmssw(JobType):
539		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
540		self.jobDestination.append(blockSites[block])
541		common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
542	+	jobsOfBlock[block].append(jobCount+1)
543		# reset counter
544		jobCount = jobCount + 1
545		totalEventCount = totalEventCount + eventsPerJobRequested
#	Line 504 | Line 560 | class Cmssw(JobType):
560		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
561		self.jobDestination.append(blockSites[block])
562		common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
563	+	jobsOfBlock[block].append(jobCount+1)
564		# increase counter
565		jobCount = jobCount + 1
566		totalEventCount = totalEventCount + eventsPerJobRequested
#	Line 521 | Line 578 | class Cmssw(JobType):
578		self.ncjobs = self.total_number_of_jobs = jobCount
579		if (eventsRemaining > 0 and jobCount < totalNumberOfJobs ):
580		common.logger.message("Could not run on all requested events because some blocks not hosted at allowed sites.")
581	<	common.logger.message("\n"+str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n")
581	>	common.logger.message(str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n")
582
583	+	# screen output
584	+	screenOutput = "List of jobs and available destination sites:\n\n"
585	+
586	+	# keep trace of block with no sites to print a warning at the end
587	+	noSiteBlock = []
588	+	bloskNoSite = []
589	+
590	+	blockCounter = 0
591	+	for block in blocks:
592	+	if block in jobsOfBlock.keys() :
593	+	blockCounter += 1
594	+	screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),','.join(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)))
595	+	if len(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)) == 0:
596	+	noSiteBlock.append( spanRanges(jobsOfBlock[block]) )
597	+	bloskNoSite.append( blockCounter )
598	+
599	+	common.logger.message(screenOutput)
600	+	if len(noSiteBlock) > 0 and len(bloskNoSite) > 0:
601	+	msg = 'WARNING: No sites are hosting any part of data for block:\n                '
602	+	virgola = ""
603	+	if len(bloskNoSite) > 1:
604	+	virgola = ","
605	+	for block in bloskNoSite:
606	+	msg += ' ' + str(block) + virgola
607	+	msg += '\n               Related jobs:\n                 '
608	+	virgola = ""
609	+	if len(noSiteBlock) > 1:
610	+	virgola = ","
611	+	for range_jobs in noSiteBlock:
612	+	msg += str(range_jobs) + virgola
613	+	msg += '\n               will not be submitted and this block of data can not be analyzed!\n'
614	+	common.logger.message(msg)
615	+
616		self.list_of_args = list_of_lists
617		return
618
#	Line 567 | Line 657 | class Cmssw(JobType):
657		## Since there is no input, any site is good
658		# self.jobDestination.append(["Any"])
659		self.jobDestination.append([""]) #must be empty to write correctly the xml
660	<	args=''
660	>	args=[]
661		if (self.firstRun):
662		## pythia first run
663		#self.list_of_args.append([(str(self.firstRun)+str(i))])
664	<	args=args+(str(self.firstRun)+str(i))
664	>	args.append(str(self.firstRun)+str(i))
665		else:
666		## no first run
667		#self.list_of_args.append([str(i)])
668	<	args=args+str(i)
668	>	args.append(str(i))
669		if (self.sourceSeed):
670	+	args.append(str(self.sourceSeed)+str(i))
671		if (self.sourceSeedVtx):
672	<	## pythia + vtx random seed
673	<	#self.list_of_args.append([
674	<	#                          str(self.sourceSeed)+str(i),
675	<	#                          str(self.sourceSeedVtx)+str(i)
676	<	#                          ])
677	<	args=args+str(',')+str(self.sourceSeed)+str(i)+str(',')+str(self.sourceSeedVtx)+str(i)
678	<	else:
679	<	## only pythia random seed
680	<	#self.list_of_args.append([(str(self.sourceSeed)+str(i))])
681	<	args=args +str(',')+str(self.sourceSeed)+str(i)
682	<	else:
683	<	## no random seed
593	<	if str(args)=='': args=args+(str(self.firstRun)+str(i))
594	<	arguments=args.split(',')
595	<	if len(arguments)==3:self.list_of_args.append([str(arguments[0]),str(arguments[1]),str(arguments[2])])
596	<	elif len(arguments)==2:self.list_of_args.append([str(arguments[0]),str(arguments[1])])
597	<	else :self.list_of_args.append([str(arguments[0])])
672	>	## + vtx random seed
673	>	args.append(str(self.sourceSeedVtx)+str(i))
674	>	if (self.sourceSeedG4):
675	>	## + G4 random seed
676	>	args.append(str(self.sourceSeedG4)+str(i))
677	>	if (self.sourceSeedMix):
678	>	## + Mix random seed
679	>	args.append(str(self.sourceSeedMix)+str(i))
680	>	pass
681	>	pass
682	>	self.list_of_args.append(args)
683	>	pass
684
685	<	#   print self.list_of_args
685	>	# print self.list_of_args
686
687		return
688
#	Line 691 | Line 777 | class Cmssw(JobType):
777		try: # create tar ball
778		tar = tarfile.open(self.tgzNameWithPath, "w:gz")
779		## First find the executable
780	<	if (executable != ''):
780	>	if (self.executable != ''):
781		exeWithPath = self.scram.findFile_(executable)
782		if ( not exeWithPath ):
783		raise CrabException('User executable '+executable+' not found')
#	Line 737 | Line 823 | class Cmssw(JobType):
823		pa = os.environ['CRABDIR'] + '/' + 'ProdAgentApi'
824		if os.path.isdir(pa):
825		tar.add(pa,paDir)
826	+
827	+	### FEDE FOR DBS PUBLICATION
828	+	## Add PRODCOMMON dir to tar
829	+	prodcommonDir = 'ProdCommon'
830	+	prodcommonPath = os.environ['CRABDIR'] + '/' + 'ProdCommon'
831	+	if os.path.isdir(prodcommonPath):
832	+	tar.add(prodcommonPath,prodcommonDir)
833	+	#############################
834
835		common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
836		tar.close()
#	Line 762 | Line 856 | class Cmssw(JobType):
856
857		return
858
859	+	def additionalInputFileTgz(self):
860	+	"""
861	+	Put all additional files into a tar ball and return its name
862	+	"""
863	+	import tarfile
864	+	tarName=  common.work_space.pathForTgz()+'share/'+self.additional_tgz_name
865	+	tar = tarfile.open(tarName, "w:gz")
866	+	for file in self.additional_inbox_files:
867	+	tar.add(file,string.split(file,'/')[-1])
868	+	common.logger.debug(5,"Files added to "+self.additional_tgz_name+" : "+str(tar.getnames()))
869	+	tar.close()
870	+	return tarName
871	+
872		def wsSetupEnvironment(self, nj):
873		"""
874		Returns part of a job script which prepares
#	Line 773 | Line 880 | class Cmssw(JobType):
880		## OLI_Daniele at this level  middleware already known
881
882		txt += 'if [ $middleware == LCG ]; then \n'
883	+	txt += '    echo "### First set SCRAM ARCH and BUILD_ARCH to ' + self.executable_arch + ' ###"\n'
884	+	txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
885	+	txt += '    export BUILD_ARCH='+self.executable_arch+'\n'
886		txt += self.wsSetupCMSLCGEnvironment_()
887		txt += 'elif [ $middleware == OSG ]; then\n'
888		txt += '    WORKING_DIR=`/bin/mktemp  -d $OSG_WN_TMP/cms_XXXXXXXXXXXX`\n'
889		txt += '    echo "Created working directory: $WORKING_DIR"\n'
890		txt += '    if [ ! -d $WORKING_DIR ] ;then\n'
891		txt += '        echo "SET_CMS_ENV 10016 ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
892	<	txt += '        echo "JOB_EXIT_STATUS = 10016"\n'
893	<	txt += '        echo "JobExitCode=10016" | tee -a $RUNTIME_AREA/$repo\n'
894	<	txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
892	>	txt += '    echo "JOB_EXIT_STATUS = 10016"\n'
893	>	txt += '    echo "JobExitCode=10016" | tee -a $RUNTIME_AREA/$repo\n'
894	>	txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
895		txt += '        rm -f $RUNTIME_AREA/$repo \n'
896		txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
897		txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
#	Line 791 | Line 901 | class Cmssw(JobType):
901		txt += '    echo "Change to working directory: $WORKING_DIR"\n'
902		txt += '    cd $WORKING_DIR\n'
903		txt += self.wsSetupCMSOSGEnvironment_()
904	+	txt += '    echo "### Set SCRAM ARCH to ' + self.executable_arch + ' ###"\n'
905	+	txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
906		txt += 'fi\n'
907
908		# Prepare JobType-specific part
#	Line 813 | Line 925 | class Cmssw(JobType):
925		txt += '        cd $RUNTIME_AREA\n'
926		txt += '        /bin/rm -rf $WORKING_DIR\n'
927		txt += '        if [ -d $WORKING_DIR ] ;then\n'
928	<	txt += '            echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
929	<	txt += '            echo "JOB_EXIT_STATUS = 10018"\n'
930	<	txt += '            echo "JobExitCode=10018" | tee -a $RUNTIME_AREA/$repo\n'
931	<	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
928	>	txt += '            echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
929	>	txt += '            echo "JOB_EXIT_STATUS = 10018"\n'
930	>	txt += '            echo "JobExitCode=10018" | tee -a $RUNTIME_AREA/$repo\n'
931	>	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
932		txt += '            rm -f $RUNTIME_AREA/$repo \n'
933		txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
934		txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
#	Line 825 | Line 937 | class Cmssw(JobType):
937		txt += '   exit 1 \n'
938		txt += 'fi \n'
939		txt += 'echo "CMSSW_VERSION =  '+self.version+'"\n'
828	–	txt += 'export SCRAM_ARCH='+self.executable_arch+'\n'
940		txt += 'cd '+self.version+'\n'
941	+	########## FEDE FOR DBS2 ######################
942	+	txt += 'SOFTWARE_DIR=`pwd`\n'
943	+	txt += 'echo SOFTWARE_DIR=$SOFTWARE_DIR \n'
944	+	###############################################
945		### needed grep for bug in scramv1 ###
946		txt += scram+' runtime -sh\n'
947		txt += 'eval `'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME`\n'
#	Line 852 | Line 967 | class Cmssw(JobType):
967		txt += '        cd $RUNTIME_AREA\n'
968		txt += '        /bin/rm -rf $WORKING_DIR\n'
969		txt += '        if [ -d $WORKING_DIR ] ;then\n'
970	<	txt += '            echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
971	<	txt += '            echo "JOB_EXIT_STATUS = 50114"\n'
972	<	txt += '            echo "JobExitCode=50114" | tee -a $RUNTIME_AREA/$repo\n'
973	<	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
970	>	txt += '            echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
971	>	txt += '            echo "JOB_EXIT_STATUS = 50114"\n'
972	>	txt += '            echo "JobExitCode=50114" | tee -a $RUNTIME_AREA/$repo\n'
973	>	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
974		txt += '            rm -f $RUNTIME_AREA/$repo \n'
975		txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
976		txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
#	Line 867 | Line 982 | class Cmssw(JobType):
982
983		# Prepare job-specific part
984		job = common.job_list[nj]
985	+	### FEDE FOR DBS OUTPUT PUBLICATION
986	+	if (self.datasetPath):
987	+	txt += '\n'
988	+	txt += 'DatasetPath='+self.datasetPath+'\n'
989	+
990	+	datasetpath_split = self.datasetPath.split("/")
991	+
992	+	txt += 'PrimaryDataset='+datasetpath_split[1]+'\n'
993	+	txt += 'DataTier='+datasetpath_split[2]+'\n'
994	+	#txt += 'ProcessedDataset='+datasetpath_split[3]+'\n'
995	+	txt += 'ApplicationFamily=cmsRun\n'
996	+
997	+	else:
998	+	txt += 'DatasetPath=MCDataTier\n'
999	+	txt += 'PrimaryDataset=null\n'
1000	+	txt += 'DataTier=null\n'
1001	+	#txt += 'ProcessedDataset=null\n'
1002	+	txt += 'ApplicationFamily=MCDataTier\n'
1003		if self.pset != None: #CarlosDaniele
1004		pset = os.path.basename(job.configFilename())
1005		txt += '\n'
1006	+	txt += 'cp  $RUNTIME_AREA/'+pset+' .\n'
1007		if (self.datasetPath): # standard job
1008		#txt += 'InputFiles=$2\n'
1009		txt += 'InputFiles=${args[1]}\n'
1010		txt += 'MaxEvents=${args[2]}\n'
1011		txt += 'SkipEvents=${args[3]}\n'
1012		txt += 'echo "Inputfiles:<$InputFiles>"\n'
1013	<	txt += 'sed "s#{\'INPUT\'}#$InputFiles#" $RUNTIME_AREA/'+pset+' > pset_tmp_1.cfg\n'
1013	>	txt += 'sed "s#{\'INPUT\'}#$InputFiles#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1014		txt += 'echo "MaxEvents:<$MaxEvents>"\n'
1015	<	txt += 'sed "s#INPUTMAXEVENTS#$MaxEvents#" pset_tmp_1.cfg > pset_tmp_2.cfg\n'
1015	>	txt += 'sed "s#INPUTMAXEVENTS#$MaxEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1016		txt += 'echo "SkipEvents:<$SkipEvents>"\n'
1017	<	txt += 'sed "s#INPUTSKIPEVENTS#$SkipEvents#" pset_tmp_2.cfg > pset.cfg\n'
1017	>	txt += 'sed "s#INPUTSKIPEVENTS#$SkipEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1018		else:  # pythia like job
1019	<	if (self.sourceSeed):
1020	<	txt += 'FirstRun=${args[1]}\n'
1019	>	seedIndex=1
1020	>	if (self.firstRun):
1021	>	txt += 'FirstRun=${args['+str(seedIndex)+']}\n'
1022		txt += 'echo "FirstRun: <$FirstRun>"\n'
1023	<	txt += 'sed "s#\<INPUTFIRSTRUN\>#$FirstRun#" $RUNTIME_AREA/'+pset+' > tmp_1.cfg\n'
1024	<	else:
1025	<	txt += '# Copy untouched pset\n'
891	<	txt += 'cp $RUNTIME_AREA/'+pset+' tmp_1.cfg\n'
1023	>	txt += 'sed "s#\<INPUTFIRSTRUN\>#$FirstRun#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1024	>	seedIndex=seedIndex+1
1025	>
1026		if (self.sourceSeed):
1027	<	#                    txt += 'Seed=$2\n'
1028	<	txt += 'Seed=${args[2]}\n'
1029	<	txt += 'echo "Seed: <$Seed>"\n'
1030	<	txt += 'sed "s#\<INPUT\>#$Seed#" tmp_1.cfg > tmp_2.cfg\n'
1027	>	txt += 'Seed=${args['+str(seedIndex)+']}\n'
1028	>	txt += 'sed "s#\<INPUT\>#$Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1029	>	seedIndex=seedIndex+1
1030	>	## the following seeds are not always present
1031		if (self.sourceSeedVtx):
1032	<	#                        txt += 'VtxSeed=$3\n'
899	<	txt += 'VtxSeed=${args[3]}\n'
1032	>	txt += 'VtxSeed=${args['+str(seedIndex)+']}\n'
1033		txt += 'echo "VtxSeed: <$VtxSeed>"\n'
1034	<	txt += 'sed "s#INPUTVTX#$VtxSeed#" tmp_2.cfg > pset.cfg\n'
1035	<	else:
1036	<	txt += 'mv tmp_2.cfg pset.cfg\n'
1037	<	else:
1038	<	txt += 'mv tmp_1.cfg pset.cfg\n'
1039	<	# txt += '# Copy untouched pset\n'
1040	<	# txt += 'cp $RUNTIME_AREA/'+pset+' pset.cfg\n'
1041	<
1034	>	txt += 'sed "s#\<INPUTVTX\>#$VtxSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1035	>	seedIndex += 1
1036	>	if (self.sourceSeedG4):
1037	>	txt += 'G4Seed=${args['+str(seedIndex)+']}\n'
1038	>	txt += 'echo "G4Seed: <$G4Seed>"\n'
1039	>	txt += 'sed "s#\<INPUTG4\>#$G4Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1040	>	seedIndex += 1
1041	>	if (self.sourceSeedMix):
1042	>	txt += 'mixSeed=${args['+str(seedIndex)+']}\n'
1043	>	txt += 'echo "MixSeed: <$mixSeed>"\n'
1044	>	txt += 'sed "s#\<INPUTMIX\>#$mixSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1045	>	seedIndex += 1
1046	>	pass
1047	>	pass
1048	>	txt += 'mv -f '+pset+' pset.cfg\n'
1049
1050		if len(self.additional_inbox_files) > 0:
1051	<	for file in self.additional_inbox_files:
1052	<	relFile = file.split("/")[-1]
1053	<	txt += 'if [ -e $RUNTIME_AREA/'+relFile+' ] ; then\n'
914	<	txt += '   cp $RUNTIME_AREA/'+relFile+' .\n'
915	<	txt += '   chmod +x '+relFile+'\n'
916	<	txt += 'fi\n'
1051	>	txt += 'if [ -e $RUNTIME_AREA/'+self.additional_tgz_name+' ] ; then\n'
1052	>	txt += '  tar xzvf $RUNTIME_AREA/'+self.additional_tgz_name+'\n'
1053	>	txt += 'fi\n'
1054		pass
1055
1056		if self.pset != None: #CarlosDaniele
#	Line 924 | Line 1061 | class Cmssw(JobType):
1061		txt += 'cat pset.cfg\n'
1062		txt += 'echo "****** end pset.cfg ********"\n'
1063		txt += '\n'
1064	+	### FEDE FOR DBS OUTPUT PUBLICATION
1065	+	txt += 'PSETHASH=`EdmConfigHash < pset.cfg` \n'
1066	+	txt += 'echo "PSETHASH = $PSETHASH" \n'
1067	+	##############
1068	+	txt += '\n'
1069		# txt += 'echo "***** cat pset1.cfg *********"\n'
1070		# txt += 'cat pset1.cfg\n'
1071		# txt += 'echo "****** end pset1.cfg ********"\n'
#	Line 965 | Line 1107 | class Cmssw(JobType):
1107		txt += '   echo "Successful untar" \n'
1108		txt += 'fi \n'
1109		txt += '\n'
1110	<	txt += 'echo "Include ProdAgentApi in PYTHONPATH"\n'
1110	>	txt += 'echo "Include ProdAgentApi and PRODCOMMON in PYTHONPATH"\n'
1111		txt += 'if [ -z "$PYTHONPATH" ]; then\n'
1112	<	txt += '   export PYTHONPATH=ProdAgentApi\n'
1112	>	#### FEDE FOR DBS OUTPUT PUBLICATION
1113	>	txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdAgentApi:$SOFTWARE_DIR/ProdCommon\n'
1114	>	#txt += '   export PYTHONPATH=`pwd`/ProdAgentApi:`pwd`/ProdCommon\n'
1115	>	#txt += '   export PYTHONPATH=ProdAgentApi\n'
1116		txt += 'else\n'
1117	<	txt += '   export PYTHONPATH=ProdAgentApi:${PYTHONPATH}\n'
1117	>	txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdAgentApi:$SOFTWARE_DIR/ProdCommon:${PYTHONPATH}\n'
1118	>	#txt += '   export PYTHONPATH=`pwd`/ProdAgentApi:`pwd`/ProdCommon:${PYTHONPATH}\n'
1119	>	#txt += '   export PYTHONPATH=ProdAgentApi:${PYTHONPATH}\n'
1120	>	txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
1121	>	###################
1122		txt += 'fi\n'
1123		txt += '\n'
1124
#	Line 992 | Line 1141 | class Cmssw(JobType):
1141		def executableArgs(self):
1142		if self.scriptExe:#CarlosDaniele
1143		return   self.scriptExe + " $NJob"
1144	<	else:
1145	<	return " -p pset.cfg"
1144	>	else:
1145	>	# if >= CMSSW_1_5_X, add -e
1146	>	version_array = self.scram.getSWVersion().split('_')
1147	>	major = 0
1148	>	minor = 0
1149	>	try:
1150	>	major = int(version_array[1])
1151	>	minor = int(version_array[2])
1152	>	except:
1153	>	msg = "Cannot parse CMSSW version string: " + "_".join(version_array) + " for major and minor release number!"
1154	>	raise CrabException(msg)
1155	>	if major >= 1 and minor >= 5 :
1156	>	return " -e -p pset.cfg"
1157	>	else:
1158	>	return " -p pset.cfg"
1159
1160		def inputSandbox(self, nj):
1161		"""
#	Line 1011 | Line 1173 | class Cmssw(JobType):
1173		if not self.pset is None:
1174		inp_box.append(common.work_space.pathForTgz() + 'job/' + self.configFilename())
1175		## additional input files
1176	<	for file in self.additional_inbox_files:
1177	<	inp_box.append(file)
1176	>	tgz = self.additionalInputFileTgz()
1177	>	inp_box.append(tgz)
1178		return inp_box
1179
1180		def outputSandbox(self, nj):
#	Line 1042 | Line 1204 | class Cmssw(JobType):
1204		txt += '# directory content\n'
1205		txt += 'ls \n'
1206
1207	<	for fileWithSuffix in (self.output_file+self.output_file_sandbox):
1207	>	txt += 'output_exit_status=0\n'
1208	>
1209	>	for fileWithSuffix in (self.output_file_sandbox):
1210		output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1211		txt += '\n'
1212		txt += '# check output file\n'
1213	<	txt += 'ls '+fileWithSuffix+'\n'
1214	<	txt += 'ls_result=$?\n'
1215	<	txt += 'if [ $ls_result -ne 0 ] ; then\n'
1216	<	txt += '   echo "ERROR: Problem with output file"\n'
1213	>	txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1214	>	txt += '    mv '+fileWithSuffix+' $RUNTIME_AREA\n'
1215	>	txt += '    cp $RUNTIME_AREA/'+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1216	>	txt += 'else\n'
1217	>	txt += '    exit_status=60302\n'
1218	>	txt += '    echo "ERROR: Problem with output file '+fileWithSuffix+'"\n'
1219		if common.scheduler.boss_scheduler_name == 'condor_g':
1220		txt += '    if [ $middleware == OSG ]; then \n'
1221		txt += '        echo "prepare dummy output file"\n'
1222		txt += '        echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
1223		txt += '    fi \n'
1224	+	txt += 'fi\n'
1225	+
1226	+	for fileWithSuffix in (self.output_file):
1227	+	output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1228	+	txt += '\n'
1229	+	txt += '# check output file\n'
1230	+	txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1231	+	txt += '    mv '+fileWithSuffix+' $RUNTIME_AREA\n'
1232	+	txt += '    cp $RUNTIME_AREA/'+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1233		txt += 'else\n'
1234	<	txt += '   cp '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1234	>	txt += '    exit_status=60302\n'
1235	>	txt += '    echo "ERROR: Problem with output file '+fileWithSuffix+'"\n'
1236	>	txt += '    echo "JOB_EXIT_STATUS = $exit_status"\n'
1237	>	txt += '    output_exit_status=$exit_status\n'
1238	>	if common.scheduler.boss_scheduler_name == 'condor_g':
1239	>	txt += '    if [ $middleware == OSG ]; then \n'
1240	>	txt += '        echo "prepare dummy output file"\n'
1241	>	txt += '        echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
1242	>	txt += '    fi \n'
1243		txt += 'fi\n'
1244	<
1245	<	txt += 'cd $RUNTIME_AREA\n'
1244	>	file_list = []
1245	>	for fileWithSuffix in (self.output_file):
1246	>	file_list.append(self.numberFile_(fileWithSuffix, '$NJob'))
1247	>
1248	>	txt += 'file_list="'+string.join(file_list,' ')+'"\n'
1249		txt += 'cd $RUNTIME_AREA\n'
1064	–	### OLI_DANIELE
1065	–	txt += 'if [ $middleware == OSG ]; then\n'
1066	–	txt += '    cd $RUNTIME_AREA\n'
1067	–	txt += '    echo "Remove working directory: $WORKING_DIR"\n'
1068	–	txt += '    /bin/rm -rf $WORKING_DIR\n'
1069	–	txt += '    if [ -d $WORKING_DIR ] ;then\n'
1070	–	txt += '        echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1071	–	txt += '        echo "JOB_EXIT_STATUS = 60999"\n'
1072	–	txt += '        echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
1073	–	txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
1074	–	txt += '        rm -f $RUNTIME_AREA/$repo \n'
1075	–	txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1076	–	txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1077	–	txt += '    fi\n'
1078	–	txt += 'fi\n'
1079	–	txt += '\n'
1080	–
1081	–	file_list = ''
1082	–	## Add to filelist only files to be possibly copied to SE
1083	–	for fileWithSuffix in self.output_file:
1084	–	output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1085	–	file_list=file_list+output_file_num+' '
1086	–	file_list=file_list[:-1]
1087	–	txt += 'file_list="'+file_list+'"\n'
1088	–
1250		return txt
1251
1252		def numberFile_(self, file, txt):
#	Line 1096 | Line 1257 | class Cmssw(JobType):
1257		# take away last extension
1258		name = p[0]
1259		for x in p[1:-1]:
1260	<	name=name+"."+x
1260	>	name=name+"."+x
1261		# add "_txt"
1262		if len(p)>1:
1263	<	ext = p[len(p)-1]
1264	<	result = name + '_' + txt + "." + ext
1263	>	ext = p[len(p)-1]
1264	>	result = name + '_' + txt + "." + ext
1265		else:
1266	<	result = name + '_' + txt
1266	>	result = name + '_' + txt
1267
1268		return result
1269
#	Line 1115 | Line 1276 | class Cmssw(JobType):
1276		req='Member("VO-cms-' + \
1277		self.version + \
1278		'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1279	+	## SL add requirement for OS version only if SL4
1280	+	#reSL4 = re.compile( r'slc4' )
1281	+	if self.executable_arch: # and reSL4.search(self.executable_arch):
1282	+	req+=' && Member("VO-cms-' + \
1283	+	self.executable_arch + \
1284	+	'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1285
1286		req = req + ' && (other.GlueHostNetworkAdapterOutboundIP)'
1287
#	Line 1134 | Line 1301 | class Cmssw(JobType):
1301		txt += '   echo "### SETUP CMS OSG  ENVIRONMENT ###"\n'
1302		txt += '   if [ -f $GRID3_APP_DIR/cmssoft/cmsset_default.sh ] ;then\n'
1303		txt += '      # Use $GRID3_APP_DIR/cmssoft/cmsset_default.sh to setup cms software\n'
1304	+	txt += '       export SCRAM_ARCH='+self.executable_arch+'\n'
1305		txt += '       source $GRID3_APP_DIR/cmssoft/cmsset_default.sh '+self.version+'\n'
1306		txt += '   elif [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n'
1307		txt += '      # Use $OSG_APP/cmssoft/cms/cmsset_default.sh to setup cms software\n'
1308	+	txt += '       export SCRAM_ARCH='+self.executable_arch+'\n'
1309		txt += '       source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1310		txt += '   else\n'
1311		txt += '       echo "SET_CMS_ENV 10020 ==> ERROR $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
#	Line 1152 | Line 1321 | class Cmssw(JobType):
1321		txt += '       cd $RUNTIME_AREA\n'
1322		txt += '       /bin/rm -rf $WORKING_DIR\n'
1323		txt += '       if [ -d $WORKING_DIR ] ;then\n'
1324	<	txt += '            echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1325	<	txt += '            echo "JOB_EXIT_STATUS = 10017"\n'
1326	<	txt += '            echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n'
1327	<	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
1328	<	txt += '            rm -f $RUNTIME_AREA/$repo \n'
1329	<	txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1330	<	txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1324	>	txt += '           echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1325	>	txt += '           echo "JOB_EXIT_STATUS = 10017"\n'
1326	>	txt += '           echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n'
1327	>	txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1328	>	txt += '           rm -f $RUNTIME_AREA/$repo \n'
1329	>	txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1330	>	txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1331		txt += '       fi\n'
1332		txt += '\n'
1333		txt += '       exit 1\n'
#	Line 1217 | Line 1386 | class Cmssw(JobType):
1386		txt += '   echo "### END SETUP CMS LCG ENVIRONMENT ###"\n'
1387		return txt
1388
1389	+	### FEDE FOR DBS OUTPUT PUBLICATION
1390	+	def modifyReport(self, nj):
1391	+	"""
1392	+	insert the part of the script that modifies the FrameworkJob Report
1393	+	"""
1394	+
1395	+	txt = ''
1396	+	try:
1397	+	publish_data = int(self.cfg_params['USER.publish_data'])
1398	+	except KeyError:
1399	+	publish_data = 0
1400	+	if (publish_data == 1):
1401	+	txt += 'echo "Modify Job Report" \n'
1402	+	#txt += 'chmod a+x $RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1403	+	################ FEDE FOR DBS2 #############################################
1404	+	txt += 'chmod a+x $SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1405	+	#############################################################################
1406	+	#try:
1407	+	#    publish_data = int(self.cfg_params['USER.publish_data'])
1408	+	#except KeyError:
1409	+	#    publish_data = 0
1410	+
1411	+	txt += 'if [ -z "$SE" ]; then\n'
1412	+	txt += '    SE="" \n'
1413	+	txt += 'fi \n'
1414	+	txt += 'if [ -z "$SE_PATH" ]; then\n'
1415	+	txt += '    SE_PATH="" \n'
1416	+	txt += 'fi \n'
1417	+	txt += 'echo "SE = $SE"\n'
1418	+	txt += 'echo "SE_PATH = $SE_PATH"\n'
1419	+
1420	+	#if (publish_data == 1):
1421	+	#processedDataset = self.cfg_params['USER.processed_datasetname']
1422	+	processedDataset = self.cfg_params['USER.publish_data_name']
1423	+	txt += 'ProcessedDataset='+processedDataset+'\n'
1424	+	#### LFN=/store/user/<user>/processedDataset_PSETHASH
1425	+	txt += 'if [ "$SE_PATH" == "" ]; then\n'
1426	+	#### FEDE: added slash in LFN ##############
1427	+	txt += '    FOR_LFN=/copy_problems/ \n'
1428	+	txt += 'else \n'
1429	+	txt += '    tmp=`echo $SE_PATH | awk -F \'store\' \'{print$2}\'` \n'
1430	+	#####  FEDE TO BE CHANGED, BECAUSE STORE IS HARDCODED!!!! ########
1431	+	txt += '    FOR_LFN=/store$tmp \n'
1432	+	txt += 'fi \n'
1433	+	txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1434	+	txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1435	+	txt += 'echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n'
1436	+	#txt += 'echo "$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1437	+	txt += 'echo "$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1438	+	txt += '$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1439	+	#txt += '$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1440	+
1441	+	txt += 'modifyReport_result=$?\n'
1442	+	txt += 'echo modifyReport_result = $modifyReport_result\n'
1443	+	txt += 'if [ $modifyReport_result -ne 0 ]; then\n'
1444	+	txt += '    exit_status=1\n'
1445	+	txt += '    echo "ERROR: Problem with ModifyJobReport"\n'
1446	+	txt += 'else\n'
1447	+	txt += '    mv NewFrameworkJobReport.xml crab_fjr_$NJob.xml\n'
1448	+	txt += 'fi\n'
1449	+	else:
1450	+	txt += 'echo "no data publication required"\n'
1451	+	#txt += 'ProcessedDataset=no_data_to_publish \n'
1452	+	#### FEDE: added slash in LFN ##############
1453	+	#txt += 'FOR_LFN=/local/ \n'
1454	+	#txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1455	+	#txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1456	+	return txt
1457	+
1458	+	def cleanEnv(self):
1459	+	### OLI_DANIELE
1460	+	txt = ''
1461	+	txt += 'if [ $middleware == OSG ]; then\n'
1462	+	txt += '    cd $RUNTIME_AREA\n'
1463	+	txt += '    echo "Remove working directory: $WORKING_DIR"\n'
1464	+	txt += '    /bin/rm -rf $WORKING_DIR\n'
1465	+	txt += '    if [ -d $WORKING_DIR ] ;then\n'
1466	+	txt += '              echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1467	+	txt += '              echo "JOB_EXIT_STATUS = 60999"\n'
1468	+	txt += '              echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
1469	+	txt += '              dumpStatus $RUNTIME_AREA/$repo\n'
1470	+	txt += '        rm -f $RUNTIME_AREA/$repo \n'
1471	+	txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1472	+	txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1473	+	txt += '    fi\n'
1474	+	txt += 'fi\n'
1475	+	txt += '\n'
1476	+	return txt
1477	+
1478		def setParam_(self, param, value):
1479		self._params[param] = value
1480
#	Line 1229 | Line 1487 | class Cmssw(JobType):
1487		def getTaskid(self):
1488		return self._taskId
1489
1232	–	#######################################################################
1490		def uniquelist(self, old):
1491		"""
1492		remove duplicates from a list
#	Line 1238 | Line 1495 | class Cmssw(JobType):
1495		for e in old:
1496		nd[e]=0
1497		return nd.keys()
1498	+
1499	+
1500	+	def checkOut(self, limit):
1501	+	"""
1502	+	check the dimension of the output files
1503	+	"""
1504	+	txt = 'echo "*****************************************"\n'
1505	+	txt += 'echo "** Starting output sandbox limit check **"\n'
1506	+	txt += 'echo "*****************************************"\n'
1507	+	allOutFiles = ""
1508	+	listOutFiles = []
1509	+	for fileOut in (self.output_file+self.output_file_sandbox):
1510	+	if fileOut.find('crab_fjr') == -1:
1511	+	allOutFiles = allOutFiles + " " + self.numberFile_(fileOut, '$NJob')
1512	+	listOutFiles.append(self.numberFile_(fileOut, '$NJob'))
1513	+	txt += 'echo "OUTPUT files: '+str(allOutFiles)+'";\n'
1514	+	txt += 'ls -gGhrta;\n'
1515	+	txt += 'sum=0;\n'
1516	+	txt += 'for file in '+str(allOutFiles)+' ; do\n'
1517	+	txt += '    if [ -e $file ]; then\n'
1518	+	txt += '        tt=`ls -gGrta $file | awk \'{ print $3 }\'`\n'
1519	+	txt += '        sum=`expr $sum + $tt`\n'
1520	+	txt += '    else\n'
1521	+	txt += '        echo "WARNING: output file $file not found!"\n'
1522	+	txt += '    fi\n'
1523	+	txt += 'done\n'
1524	+	txt += 'echo "Total Output dimension: $sum";\n'
1525	+	txt += 'limit='+str(limit)+';\n'
1526	+	txt += 'echo "OUTPUT FILES LIMIT SET TO: $limit";\n'
1527	+	txt += 'if [ $limit -lt $sum ]; then\n'
1528	+	txt += '    echo "WARNING: output files have to big size - something will be lost;"\n'
1529	+	txt += '    echo "         checking the output file sizes..."\n'
1530	+	"""
1531	+	txt += '    dim=0;\n'
1532	+	txt += '    exclude=0;\n'
1533	+	txt += '    for files in '+str(allOutFiles)+' ; do\n'
1534	+	txt += '        sumTemp=0;\n'
1535	+	txt += '        for file2 in '+str(allOutFiles)+' ; do\n'
1536	+	txt += '            if [ $file != $file2 ]; then\n'
1537	+	txt += '                tt=`ls -gGrta $file2 | awk \'{ print $3 }\';`\n'
1538	+	txt += '                sumTemp=`expr $sumTemp + $tt`;\n'
1539	+	txt += '            fi\n'
1540	+	txt += '        done\n'
1541	+	txt += '        if [ $sumTemp -lt $limit ]; then\n'
1542	+	txt += '            if [ $dim -lt $sumTemp ]; then\n'
1543	+	txt += '                dim=$sumTemp;\n'
1544	+	txt += '                exclude=$file;\n'
1545	+	txt += '            fi\n'
1546	+	txt += '        fi\n'
1547	+	txt += '    done\n'
1548	+	txt += '    echo "Dimension calculated: $dim"; echo "File to exclude: $exclude";\n'
1549	+	"""
1550	+	txt += '    tot=0;\n'
1551	+	txt += '    for file2 in '+str(allOutFiles)+' ; do\n'
1552	+	txt += '        tt=`ls -gGrta $file2 | awk \'{ print $3 }\';`\n'
1553	+	txt += '        tot=`expr $tot + $tt`;\n'
1554	+	txt += '        if [ $limit -lt $tot ]; then\n'
1555	+	txt += '            tot=`expr $tot - $tt`;\n'
1556	+	txt += '            fileLast=$file;\n'
1557	+	txt += '            break;\n'
1558	+	txt += '        fi\n'
1559	+	txt += '    done\n'
1560	+	txt += '    echo "Dimension calculated: $tot"; echo "First file to exclude: $file";\n'
1561	+	txt += '    flag=0;\n'
1562	+	txt += '    for filess in '+str(allOutFiles)+' ; do\n'
1563	+	txt += '        if [ $fileLast = $filess ]; then\n'
1564	+	txt += '            flag=1;\n'
1565	+	txt += '        fi\n'
1566	+	txt += '        if [ $flag -eq 1 ]; then\n'
1567	+	txt += '            rm -f $filess;\n'
1568	+	txt += '        fi\n'
1569	+	txt += '    done\n'
1570	+	txt += '    ls -agGhrt;\n'
1571	+	txt += '    echo "WARNING: output files are too big in dimension: can not put in the output_sandbox.";\n'
1572	+	txt += '    echo "JOB_EXIT_STATUS = 70000";\n'
1573	+	txt += '    exit_status=70000;\n'
1574	+	txt += 'else'
1575	+	txt += '    echo "Total Output dimension $sum is fine.";\n'
1576	+	txt += 'fi\n'
1577	+	txt += 'echo "*****************************************"\n'
1578	+	txt += 'echo "*** Ending output sandbox limit check ***"\n'
1579	+	txt += 'echo "*****************************************"\n'
1580	+	return txt

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines