ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/cms_cmssw.py

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.117 by fanzago, Fri Aug 17 10:45:26 2007 UTC vs.
Revision 1.132 by ewv, Mon Oct 29 21:03:11 2007 UTC

#	Line 28 | Line 28 | class Cmssw(JobType):
28		self.ncjobs = ncjobs
29
30		log = common.logger
31	<
31	>
32		self.scram = Scram.Scram(cfg_params)
33		self.additional_inbox_files = []
34		self.scriptExe = ''
#	Line 37 | Line 37 | class Cmssw(JobType):
37		self.tgz_name = 'default.tgz'
38		self.additional_tgz_name = 'additional.tgz'
39		self.scriptName = 'CMSSW.sh'
40	<	self.pset = ''      #scrip use case Da
40	>	self.pset = ''      #scrip use case Da
41		self.datasetPath = '' #scrip use case Da
42
43		# set FJR file name
44		self.fjrFileName = 'crab_fjr.xml'
45
46		self.version = self.scram.getSWVersion()
47	<
47	>
48		#
49		# Try to block creation in case of arch/version mismatch
50		#
#	Line 57 | Line 57 | class Cmssw(JobType):
57		if int(a[1]) == 1 and (int(a[2]) >= 5 and self.executable_arch.find('slc3') == 0):
58		msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch)
59		raise CrabException(msg)
60	<
60	>
61		common.taskDB.setDict('codeVersion',self.version)
62		self.setParam_('application', self.version)
63
#	Line 68 | Line 68 | class Cmssw(JobType):
68		self.use_dbs_1 = int(self.cfg_params['CMSSW.use_dbs_1'])
69		except KeyError:
70		self.use_dbs_1 = 0
71	<
71	>
72		try:
73		tmp =  cfg_params['CMSSW.datasetpath']
74		log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
#	Line 79 | Line 79 | class Cmssw(JobType):
79		self.datasetPath = tmp
80		self.selectNoInput = 0
81		except KeyError:
82	<	msg = "Error: datasetpath not defined "
82	>	msg = "Error: datasetpath not defined "
83		raise CrabException(msg)
84
85		# ML monitoring
#	Line 91 | Line 91 | class Cmssw(JobType):
91		try:
92		datasetpath_split = self.datasetPath.split("/")
93		# standard style
94	+	self.setParam_('datasetFull', self.datasetPath)
95		if self.use_dbs_1 == 1 :
96		self.setParam_('dataset', datasetpath_split[1])
97		self.setParam_('owner', datasetpath_split[-1])
#	Line 100 | Line 101 | class Cmssw(JobType):
101		except:
102		self.setParam_('dataset', self.datasetPath)
103		self.setParam_('owner', self.datasetPath)
104	<
104	>
105		self.setTaskid_()
106		self.setParam_('taskId', self.cfg_params['taskId'])
107
#	Line 123 | Line 124 | class Cmssw(JobType):
124		try:
125		self.pset = cfg_params['CMSSW.pset']
126		log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset)
127	<	if self.pset.lower() != 'none' :
127	>	if self.pset.lower() != 'none' :
128		if (not os.path.exists(self.pset)):
129		raise CrabException("User defined PSet file "+self.pset+" does not exist")
130		else:
#	Line 215 | Line 216 | class Cmssw(JobType):
216		except KeyError:
217		self.eventsPerJob = -1
218		self.selectEventsPerJob = 0
219	<
219	>
220		## number of jobs
221		try:
222		self.theNumberOfJobs =int( cfg_params['CMSSW.number_of_jobs'])
#	Line 231 | Line 232 | class Cmssw(JobType):
232		self.total_number_of_events = 0
233		self.selectTotalNumberEvents = 0
234
235	<	if self.pset != None: #CarlosDaniele
235	>	if self.pset != None: #CarlosDaniele
236		if ( (self.selectTotalNumberEvents + self.selectEventsPerJob + self.selectNumberOfJobs) != 2 ):
237		msg = 'Must define exactly two of total_number_of_events, events_per_job, or number_of_jobs.'
238		raise CrabException(msg)
#	Line 271 | Line 272 | class Cmssw(JobType):
272		self.firstRun = None
273		common.logger.debug(5,"No first run given")
274		if self.pset != None: #CarlosDaniele
275	<	ver = string.split(self.version,"_")
275	<	if (int(ver[1])>=1 and int(ver[2])>=5):
276	<	import PsetManipulator150 as pp
277	<	else:
278	<	import PsetManipulator as pp
275	>	import PsetManipulator as pp
276		PsetEdit = pp.PsetManipulator(self.pset) #Daniele Pset
277
278		#DBSDLS-start
279	<	## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
279	>	## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
280		self.maxEvents=0  # max events available   ( --> check the requested nb. of evts in Creator.py)
281		self.DBSPaths={}  # all dbs paths requested ( --> input to the site local discovery script)
282		self.jobDestination=[]  # Site destination(s) for each job (list of lists)
#	Line 288 | Line 285 | class Cmssw(JobType):
285		blockSites = {}
286		if self.datasetPath:
287		blockSites = self.DataDiscoveryAndLocation(cfg_params)
288	<	#DBSDLS-end
288	>	#DBSDLS-end
289
290		self.tgzNameWithPath = self.getTarBall(self.executable)
291	<
291	>
292		## Select Splitting
293	<	if self.selectNoInput:
293	>	if self.selectNoInput:
294		if self.pset == None: #CarlosDaniele
295		self.jobSplittingForScript()
296		else:
#	Line 306 | Line 303 | class Cmssw(JobType):
303		try:
304		if (self.datasetPath): # standard job
305		# allow to processa a fraction of events in a file
306	<	PsetEdit.inputModule("INPUT")
307	<	PsetEdit.maxEvent("INPUTMAXEVENTS")
308	<	PsetEdit.skipEvent("INPUTSKIPEVENTS")
306	>	PsetEdit.inputModule("INPUTFILE")
307	>	PsetEdit.maxEvent(0)
308	>	PsetEdit.skipEvent(0)
309		else:  # pythia like job
310		PsetEdit.maxEvent(self.eventsPerJob)
311		if (self.firstRun):
312	<	PsetEdit.pythiaFirstRun("INPUTFIRSTRUN")  #First Run
312	>	PsetEdit.pythiaFirstRun(0)  #First Run
313		if (self.sourceSeed) :
314	<	PsetEdit.pythiaSeed("INPUT")
314	>	PsetEdit.pythiaSeed(0)
315		if (self.sourceSeedVtx) :
316	<	PsetEdit.vtxSeed("INPUTVTX")
316	>	PsetEdit.vtxSeed(0)
317		if (self.sourceSeedG4) :
318	<	self.PsetEdit.g4Seed("INPUTG4")
318	>	PsetEdit.g4Seed(0)
319		if (self.sourceSeedMix) :
320	<	self.PsetEdit.mixSeed("INPUTMIX")
320	>	PsetEdit.mixSeed(0)
321		# add FrameworkJobReport to parameter-set
322		PsetEdit.addCrabFJR(self.fjrFileName)
323		PsetEdit.psetWriter(self.configFilename())
#	Line 371 | Line 368 | class Cmssw(JobType):
368		self.eventsbyfile=self.pubdata.getEventsPerFile()
369
370		## get max number of events
371	<	self.maxEvents=self.pubdata.getMaxEvents() ##  self.maxEvents used in Creator.py
371	>	self.maxEvents=self.pubdata.getMaxEvents() ##  self.maxEvents used in Creator.py
372
373		## Contact the DLS and build a list of sites hosting the fileblocks
374		try:
#	Line 380 | Line 377 | class Cmssw(JobType):
377		except DataLocation.DataLocationError , ex:
378		msg = 'ERROR ***: failed Data Location in DLS \n %s '%ex.getErrorMessage()
379		raise CrabException(msg)
380	<
380	>
381
382		sites = dataloc.getSites()
383		allSites = []
#	Line 394 | Line 391 | class Cmssw(JobType):
391		common.logger.message("Requested dataset: " + datasetPath + " has " + str(self.maxEvents) + " events in " + str(len(self.filesbyblock.keys())) + " blocks.\n")
392
393		return sites
394	<
394	>
395		def jobSplittingByBlocks(self, blockSites):
396		"""
397		Perform job splitting. Jobs run over an integer number of files
#	Line 444 | Line 441 | class Cmssw(JobType):
441		totalNumberOfJobs = 999999999
442		else :
443		totalNumberOfJobs = self.ncjobs
444	<
444	>
445
446		blocks = blockSites.keys()
447		blockCount = 0
#	Line 464 | Line 461 | class Cmssw(JobType):
461		blockCount += 1
462		if block not in jobsOfBlock.keys() :
463		jobsOfBlock[block] = []
464	<
464	>
465		if self.eventsbyblock.has_key(block) :
466		numEventsInBlock = self.eventsbyblock[block]
467		common.logger.debug(5,'Events in Block File '+str(numEventsInBlock))
468	<
468	>
469		files = self.filesbyblock[block]
470		numFilesInBlock = len(files)
471		if (numFilesInBlock <= 0):
#	Line 476 | Line 473 | class Cmssw(JobType):
473		fileCount = 0
474
475		# ---- New block => New job ---- #
476	<	parString = "\\{"
476	>	parString = ""
477		# counter for number of events in files currently worked on
478		filesEventCount = 0
479		# flag if next while loop should touch new file
480		newFile = 1
481		# job event counter
482		jobSkipEventCount = 0
483	<
483	>
484		# ---- Iterate over the files in the block until we've met the requested ---- #
485		# ---- total # of events or we've gone over all the files in this block  ---- #
486		while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) and (jobCount < totalNumberOfJobs) ):
#	Line 499 | Line 496 | class Cmssw(JobType):
496		newFile = 0
497		except KeyError:
498		common.logger.message("File "+str(file)+" has unknown number of events: skipping")
499	<
499	>
500
501		# if less events in file remain than eventsPerJobRequested
502		if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested ) :
#	Line 508 | Line 505 | class Cmssw(JobType):
505		# end job using last file, use remaining events in block
506		# close job and touch new file
507		fullString = parString[:-2]
511	–	fullString += '\\}'
508		list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
509		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(filesEventCount - jobSkipEventCount)+" events (last file in block).")
510		self.jobDestination.append(blockSites[block])
#	Line 521 | Line 517 | class Cmssw(JobType):
517		eventsRemaining = eventsRemaining - filesEventCount + jobSkipEventCount
518		jobSkipEventCount = 0
519		# reset file
520	<	parString = "\\{"
520	>	parString = ""
521		filesEventCount = 0
522		newFile = 1
523		fileCount += 1
#	Line 533 | Line 529 | class Cmssw(JobType):
529		elif ( filesEventCount - jobSkipEventCount == eventsPerJobRequested ) :
530		# close job and touch new file
531		fullString = parString[:-2]
536	–	fullString += '\\}'
532		list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
533		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
534		self.jobDestination.append(blockSites[block])
#	Line 545 | Line 540 | class Cmssw(JobType):
540		eventsRemaining = eventsRemaining - eventsPerJobRequested
541		jobSkipEventCount = 0
542		# reset file
543	<	parString = "\\{"
543	>	parString = ""
544		filesEventCount = 0
545		newFile = 1
546		fileCount += 1
547	<
547	>
548		# if more events in file remain than eventsPerJobRequested
549		else :
550		# close job but don't touch new file
551		fullString = parString[:-2]
557	–	fullString += '\\}'
552		list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
553		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
554		self.jobDestination.append(blockSites[block])
#	Line 569 | Line 563 | class Cmssw(JobType):
563		jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file])
564		# remove all but the last file
565		filesEventCount = self.eventsbyfile[file]
566	<	parString = "\\{"
566	>	parString = ""
567		parString += '\\\"' + file + '\\\"\,'
568		pass # END if
569		pass # END while (iterate over files in the block)
#	Line 578 | Line 572 | class Cmssw(JobType):
572		if (eventsRemaining > 0 and jobCount < totalNumberOfJobs ):
573		common.logger.message("Could not run on all requested events because some blocks not hosted at allowed sites.")
574		common.logger.message(str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n")
575	<
575	>
576		# screen output
577		screenOutput = "List of jobs and available destination sites:\n\n"
578
579	+	# keep trace of block with no sites to print a warning at the end
580	+	noSiteBlock = []
581	+	bloskNoSite = []
582	+
583		blockCounter = 0
584		for block in blocks:
585		if block in jobsOfBlock.keys() :
586		blockCounter += 1
587		screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),','.join(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)))
588	+	if len(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)) == 0:
589	+	noSiteBlock.append( spanRanges(jobsOfBlock[block]) )
590	+	bloskNoSite.append( blockCounter )
591
592		common.logger.message(screenOutput)
593	+	if len(noSiteBlock) > 0 and len(bloskNoSite) > 0:
594	+	msg = 'WARNING: No sites are hosting any part of data for block:\n                '
595	+	virgola = ""
596	+	if len(bloskNoSite) > 1:
597	+	virgola = ","
598	+	for block in bloskNoSite:
599	+	msg += ' ' + str(block) + virgola
600	+	msg += '\n               Related jobs:\n                 '
601	+	virgola = ""
602	+	if len(noSiteBlock) > 1:
603	+	virgola = ","
604	+	for range_jobs in noSiteBlock:
605	+	msg += str(range_jobs) + virgola
606	+	msg += '\n               will not be submitted and this block of data can not be analyzed!\n'
607	+	common.logger.message(msg)
608
609		self.list_of_args = list_of_lists
610		return
#	Line 598 | Line 614 | class Cmssw(JobType):
614		Perform job splitting based on number of event per job
615		"""
616		common.logger.debug(5,'Splitting per events')
617	<	common.logger.message('Required '+str(self.eventsPerJob)+' events per job ')
618	<	common.logger.message('Required '+str(self.theNumberOfJobs)+' jobs in total ')
619	<	common.logger.message('Required '+str(self.total_number_of_events)+' events in total ')
617	>
618	>	if (self.selectEventsPerJob):
619	>	common.logger.message('Required '+str(self.eventsPerJob)+' events per job ')
620	>	if (self.selectNumberOfJobs):
621	>	common.logger.message('Required '+str(self.theNumberOfJobs)+' jobs in total ')
622	>	if (self.selectTotalNumberEvents):
623	>	common.logger.message('Required '+str(self.total_number_of_events)+' events in total ')
624
625		if (self.total_number_of_events < 0):
626		msg='Cannot split jobs per Events with "-1" as total number of events'
#	Line 609 | Line 629 | class Cmssw(JobType):
629		if (self.selectEventsPerJob):
630		if (self.selectTotalNumberEvents):
631		self.total_number_of_jobs = int(self.total_number_of_events/self.eventsPerJob)
632	<	elif(self.selectNumberOfJobs) :
632	>	elif(self.selectNumberOfJobs) :
633		self.total_number_of_jobs =self.theNumberOfJobs
634	<	self.total_number_of_events =int(self.theNumberOfJobs*self.eventsPerJob)
634	>	self.total_number_of_events =int(self.theNumberOfJobs*self.eventsPerJob)
635
636		elif (self.selectNumberOfJobs) :
637		self.total_number_of_jobs = self.theNumberOfJobs
638		self.eventsPerJob = int(self.total_number_of_events/self.total_number_of_jobs)
639	<
639	>
640		common.logger.debug(5,'N jobs  '+str(self.total_number_of_jobs))
641
642		# is there any remainder?
#	Line 633 | Line 653 | class Cmssw(JobType):
653		for i in range(self.total_number_of_jobs):
654		## Since there is no input, any site is good
655		# self.jobDestination.append(["Any"])
656	<	self.jobDestination.append([""]) #must be empty to write correctly the xml
656	>	self.jobDestination.append([""]) #must be empty to write correctly the xml
657		args=[]
658		if (self.firstRun):
659		## pythia first run
#	Line 651 | Line 671 | class Cmssw(JobType):
671		if (self.sourceSeedG4):
672		## + G4 random seed
673		args.append(str(self.sourceSeedG4)+str(i))
674	<	if (self.sourceSeedMix):
674	>	if (self.sourceSeedMix):
675		## + Mix random seed
676		args.append(str(self.sourceSeedMix)+str(i))
677		pass
678		pass
679		self.list_of_args.append(args)
680		pass
681	<
681	>
682		# print self.list_of_args
683
684		return
#	Line 688 | Line 708 | class Cmssw(JobType):
708		return
709
710		def split(self, jobParams):
711	<
711	>
712		common.jobDB.load()
713		#### Fabio
714		njobs = self.total_number_of_jobs
#	Line 696 | Line 716 | class Cmssw(JobType):
716		# create the empty structure
717		for i in range(njobs):
718		jobParams.append("")
719	<
719	>
720		for job in range(njobs):
721		jobParams[job] = arglist[job]
722		# print str(arglist[job])
#	Line 707 | Line 727 | class Cmssw(JobType):
727
728		common.jobDB.save()
729		return
730	<
730	>
731		def getJobTypeArguments(self, nj, sched):
732		result = ''
733		for i in common.jobDB.arguments(nj):
734		result=result+str(i)+" "
735		return result
736	<
736	>
737		def numberOfJobs(self):
738		# Fabio
739		return self.total_number_of_jobs
#	Line 722 | Line 742 | class Cmssw(JobType):
742		"""
743		Return the TarBall with lib and exe
744		"""
745	<
745	>
746		# if it exist, just return it
747		#
748		# Marco. Let's start to use relative path for Boss XML files
#	Line 745 | Line 765 | class Cmssw(JobType):
765		# print "swVersion = ", swVersion
766		swReleaseTop = self.scram.getReleaseTop_()
767		#print "swReleaseTop = ", swReleaseTop
768	<
768	>
769		## check if working area is release top
770		if swReleaseTop == '' or swArea == swReleaseTop:
771		return
#	Line 758 | Line 778 | class Cmssw(JobType):
778		exeWithPath = self.scram.findFile_(executable)
779		if ( not exeWithPath ):
780		raise CrabException('User executable '+executable+' not found')
781	<
781	>
782		## then check if it's private or not
783		if exeWithPath.find(swReleaseTop) == -1:
784		# the exe is private, so we must ship
#	Line 767 | Line 787 | class Cmssw(JobType):
787		# distinguish case when script is in user project area or given by full path somewhere else
788		if exeWithPath.find(path) >= 0 :
789		exe = string.replace(exeWithPath, path,'')
790	<	tar.add(path+exe,os.path.basename(executable))
790	>	tar.add(path+exe,exe)
791		else :
792		tar.add(exeWithPath,os.path.basename(executable))
793		pass
794		else:
795		# the exe is from release, we'll find it on WN
796		pass
797	<
797	>
798		## Now get the libraries: only those in local working area
799		libDir = 'lib'
800		lib = swArea+'/' +libDir
801		common.logger.debug(5,"lib "+lib+" to be tarred")
802		if os.path.exists(lib):
803		tar.add(lib,libDir)
804	<
804	>
805		## Now check if module dir is present
806		moduleDir = 'module'
807		module = swArea + '/' + moduleDir
#	Line 807 | Line 827 | class Cmssw(JobType):
827		prodcommonPath = os.environ['CRABDIR'] + '/' + 'ProdCommon'
828		if os.path.isdir(prodcommonPath):
829		tar.add(prodcommonPath,prodcommonDir)
830	<	#############################
831	<
830	>	#############################
831	>
832		common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
833		tar.close()
834		except :
#	Line 820 | Line 840 | class Cmssw(JobType):
840		raise CrabException('Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' + str(self.MaxTarBallSize) + ' MB input sandbox limit and not supported by the used GRID submission system. Please make sure that no unnecessary files are in all data directories in your local CMSSW project area as they are automatically packed into the input sandbox.')
841
842		## create tar-ball with ML stuff
843	<	self.MLtgzfile =  common.work_space.pathForTgz()+'share/MLfiles.tgz'
843	>	self.MLtgzfile =  common.work_space.pathForTgz()+'share/MLfiles.tgz'
844		try:
845		tar = tarfile.open(self.MLtgzfile, "w:gz")
846		path=os.environ['CRABDIR'] + '/python/'
#	Line 830 | Line 850 | class Cmssw(JobType):
850		tar.close()
851		except :
852		raise CrabException('Could not create ML files tar-ball')
853	<
853	>
854		return
855	<
855	>
856		def additionalInputFileTgz(self):
857		"""
858		Put all additional files into a tar ball and return its name
#	Line 852 | Line 872 | class Cmssw(JobType):
872		the execution environment for the job 'nj'.
873		"""
874		# Prepare JobType-independent part
875	<	txt = ''
876	<
875	>	txt = ''
876	>
877		## OLI_Daniele at this level  middleware already known
878
879	<	txt += 'if [ $middleware == LCG ]; then \n'
879	>	txt += 'if [ $middleware == LCG ]; then \n'
880		txt += '    echo "### First set SCRAM ARCH and BUILD_ARCH to ' + self.executable_arch + ' ###"\n'
881		txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
882		txt += '    export BUILD_ARCH='+self.executable_arch+'\n'
883		txt += self.wsSetupCMSLCGEnvironment_()
884		txt += 'elif [ $middleware == OSG ]; then\n'
885		txt += '    WORKING_DIR=`/bin/mktemp  -d $OSG_WN_TMP/cms_XXXXXXXXXXXX`\n'
886	<	txt += '    echo "Created working directory: $WORKING_DIR"\n'
867	<	txt += '    if [ ! -d $WORKING_DIR ] ;then\n'
886	>	txt += '    if [ ! $? == 0 ] ;then\n'
887		txt += '        echo "SET_CMS_ENV 10016 ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
888		txt += '    echo "JOB_EXIT_STATUS = 10016"\n'
889		txt += '    echo "JobExitCode=10016" | tee -a $RUNTIME_AREA/$repo\n'
#	Line 874 | Line 893 | class Cmssw(JobType):
893		txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
894		txt += '        exit 1\n'
895		txt += '    fi\n'
896	+	txt += '    echo "Created working directory: $WORKING_DIR"\n'
897		txt += '\n'
898		txt += '    echo "Change to working directory: $WORKING_DIR"\n'
899		txt += '    cd $WORKING_DIR\n'
900	<	txt += self.wsSetupCMSOSGEnvironment_()
900	>	txt += self.wsSetupCMSOSGEnvironment_()
901		txt += '    echo "### Set SCRAM ARCH to ' + self.executable_arch + ' ###"\n'
902		txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
903		txt += 'fi\n'
#	Line 960 | Line 980 | class Cmssw(JobType):
980		# Prepare job-specific part
981		job = common.job_list[nj]
982		### FEDE FOR DBS OUTPUT PUBLICATION
983	<	if (self.datasetPath):
983	>	if (self.datasetPath):
984		txt += '\n'
985		txt += 'DatasetPath='+self.datasetPath+'\n'
986
987		datasetpath_split = self.datasetPath.split("/")
988	<
988	>
989		txt += 'PrimaryDataset='+datasetpath_split[1]+'\n'
990		txt += 'DataTier='+datasetpath_split[2]+'\n'
991		#txt += 'ProcessedDataset='+datasetpath_split[3]+'\n'
#	Line 987 | Line 1007 | class Cmssw(JobType):
1007		txt += 'MaxEvents=${args[2]}\n'
1008		txt += 'SkipEvents=${args[3]}\n'
1009		txt += 'echo "Inputfiles:<$InputFiles>"\n'
1010	<	txt += 'sed "s#{\'INPUT\'}#$InputFiles#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1010	>	txt += 'sed "s#\'INPUTFILE\'#$InputFiles#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1011		txt += 'echo "MaxEvents:<$MaxEvents>"\n'
1012	<	txt += 'sed "s#INPUTMAXEVENTS#$MaxEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1012	>	txt += 'sed "s#int32 input = 0#int32 input = $MaxEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1013		txt += 'echo "SkipEvents:<$SkipEvents>"\n'
1014	<	txt += 'sed "s#INPUTSKIPEVENTS#$SkipEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1014	>	txt += 'sed "s#uint32 skipEvents = 0#uint32 skipEvents = $SkipEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1015		else:  # pythia like job
1016		seedIndex=1
1017		if (self.firstRun):
1018		txt += 'FirstRun=${args['+str(seedIndex)+']}\n'
1019		txt += 'echo "FirstRun: <$FirstRun>"\n'
1020	<	txt += 'sed "s#\<INPUTFIRSTRUN\>#$FirstRun#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1020	>	txt += 'sed "s#uint32 firstRun = 0#uint32 firstRun = $FirstRun#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1021		seedIndex=seedIndex+1
1022
1023		if (self.sourceSeed):
1024		txt += 'Seed=${args['+str(seedIndex)+']}\n'
1025	<	txt += 'sed "s#\<INPUT\>#$Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1025	>	txt += 'sed "s#uint32 sourceSeed = 0#uint32 sourceSeed = $Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1026		seedIndex=seedIndex+1
1027		## the following seeds are not always present
1028		if (self.sourceSeedVtx):
1029		txt += 'VtxSeed=${args['+str(seedIndex)+']}\n'
1030		txt += 'echo "VtxSeed: <$VtxSeed>"\n'
1031	<	txt += 'sed "s#\<INPUTVTX\>#$VtxSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1031	>	txt += 'sed "s#uint32 VtxSmeared = 0#uint32 VtxSmeared = $VtxSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1032		seedIndex += 1
1033		if (self.sourceSeedG4):
1034		txt += 'G4Seed=${args['+str(seedIndex)+']}\n'
1035		txt += 'echo "G4Seed: <$G4Seed>"\n'
1036	<	txt += 'sed "s#\<INPUTG4\>#$G4Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1036	>	txt += 'sed "s#uint32 g4SimHits = 0#uint32 g4SimHits = $G4Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1037		seedIndex += 1
1038		if (self.sourceSeedMix):
1039		txt += 'mixSeed=${args['+str(seedIndex)+']}\n'
1040		txt += 'echo "MixSeed: <$mixSeed>"\n'
1041	<	txt += 'sed "s#\<INPUTMIX\>#$mixSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1041	>	txt += 'sed "s#uint32 mix = 0#uint32 mix = $mixSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1042		seedIndex += 1
1043		pass
1044		pass
#	Line 1028 | Line 1048 | class Cmssw(JobType):
1048		txt += 'if [ -e $RUNTIME_AREA/'+self.additional_tgz_name+' ] ; then\n'
1049		txt += '  tar xzvf $RUNTIME_AREA/'+self.additional_tgz_name+'\n'
1050		txt += 'fi\n'
1051	<	pass
1051	>	pass
1052
1053		if self.pset != None: #CarlosDaniele
1054		txt += 'echo "### END JOB SETUP ENVIRONMENT ###"\n\n'
1055	<
1055	>
1056		txt += '\n'
1057		txt += 'echo "***** cat pset.cfg *********"\n'
1058		txt += 'cat pset.cfg\n'
#	Line 1041 | Line 1061 | class Cmssw(JobType):
1061		### FEDE FOR DBS OUTPUT PUBLICATION
1062		txt += 'PSETHASH=`EdmConfigHash < pset.cfg` \n'
1063		txt += 'echo "PSETHASH = $PSETHASH" \n'
1064	<	##############
1064	>	##############
1065		txt += '\n'
1066		# txt += 'echo "***** cat pset1.cfg *********"\n'
1067		# txt += 'cat pset1.cfg\n'
#	Line 1095 | Line 1115 | class Cmssw(JobType):
1115		#txt += '   export PYTHONPATH=`pwd`/ProdAgentApi:`pwd`/ProdCommon:${PYTHONPATH}\n'
1116		#txt += '   export PYTHONPATH=ProdAgentApi:${PYTHONPATH}\n'
1117		txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
1118	<	###################
1118	>	###################
1119		txt += 'fi\n'
1120		txt += '\n'
1121
1122		pass
1123	<
1123	>
1124		return txt
1125
1126		def modifySteeringCards(self, nj):
1127		"""
1128	<	modify the card provided by the user,
1128	>	modify the card provided by the user,
1129		writing a new card into share dir
1130		"""
1131	<
1131	>
1132		def executableName(self):
1133		if self.scriptExe: #CarlosDaniele
1134		return "sh "
#	Line 1127 | Line 1147 | class Cmssw(JobType):
1147		major = int(version_array[1])
1148		minor = int(version_array[2])
1149		except:
1150	<	msg = "Cannot parse CMSSW version string: " + "_".join(version_array) + " for major and minor release number!"
1150	>	msg = "Cannot parse CMSSW version string: " + "_".join(version_array) + " for major and minor release number!"
1151		raise CrabException(msg)
1152		if major >= 1 and minor >= 5 :
1153		return " -e -p pset.cfg"
#	Line 1162 | Line 1182 | class Cmssw(JobType):
1182
1183		## User Declared output files
1184		for out in (self.output_file+self.output_file_sandbox):
1185	<	n_out = nj + 1
1185	>	n_out = nj + 1
1186		out_box.append(self.numberFile_(out,str(n_out)))
1187		return out_box
1188
#	Line 1181 | Line 1201 | class Cmssw(JobType):
1201		txt += '# directory content\n'
1202		txt += 'ls \n'
1203
1204	<	for fileWithSuffix in (self.output_file+self.output_file_sandbox):
1204	>	txt += 'output_exit_status=0\n'
1205	>
1206	>	for fileWithSuffix in (self.output_file_sandbox):
1207	>	output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1208	>	txt += '\n'
1209	>	txt += '# check output file\n'
1210	>	txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1211	>	txt += '    mv '+fileWithSuffix+' $RUNTIME_AREA\n'
1212	>	txt += '    cp $RUNTIME_AREA/'+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1213	>	txt += 'else\n'
1214	>	txt += '    exit_status=60302\n'
1215	>	txt += '    echo "ERROR: Problem with output file '+fileWithSuffix+'"\n'
1216	>	if common.scheduler.boss_scheduler_name == 'condor_g':
1217	>	txt += '    if [ $middleware == OSG ]; then \n'
1218	>	txt += '        echo "prepare dummy output file"\n'
1219	>	txt += '        echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
1220	>	txt += '    fi \n'
1221	>	txt += 'fi\n'
1222	>
1223	>	for fileWithSuffix in (self.output_file):
1224		output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1225		txt += '\n'
1226		txt += '# check output file\n'
1188	–	# txt += 'ls '+fileWithSuffix+'\n'
1189	–	# txt += 'ls_result=$?\n'
1227		txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1191	–	###### FEDE FOR OUTPUT DATA PUBLICATION ########
1228		txt += '    mv '+fileWithSuffix+' $RUNTIME_AREA\n'
1229		txt += '    cp $RUNTIME_AREA/'+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1194	–	################################################
1230		txt += 'else\n'
1231		txt += '    exit_status=60302\n'
1232		txt += '    echo "ERROR: Problem with output file '+fileWithSuffix+'"\n'
1233	<	############# FEDE ADDED CHECK FOR OUTPUT #############
1234	<	if fileWithSuffix in self.output_file:
1200	<	txt += '    echo "JOB_EXIT_STATUS = $exit_status"\n'
1201	<	txt += '    exit $exit_status\n'
1202	<	#######################################################
1233	>	txt += '    echo "JOB_EXIT_STATUS = $exit_status"\n'
1234	>	txt += '    output_exit_status=$exit_status\n'
1235		if common.scheduler.boss_scheduler_name == 'condor_g':
1236		txt += '    if [ $middleware == OSG ]; then \n'
1237		txt += '        echo "prepare dummy output file"\n'
#	Line 1209 | Line 1241 | class Cmssw(JobType):
1241		file_list = []
1242		for fileWithSuffix in (self.output_file):
1243		file_list.append(self.numberFile_(fileWithSuffix, '$NJob'))
1244	<
1244	>
1245		txt += 'file_list="'+string.join(file_list,' ')+'"\n'
1246		txt += 'cd $RUNTIME_AREA\n'
1247		return txt
#	Line 1229 | Line 1261 | class Cmssw(JobType):
1261		result = name + '_' + txt + "." + ext
1262		else:
1263		result = name + '_' + txt
1264	<
1264	>
1265		return result
1266
1267		def getRequirements(self, nj=[]):
1268		"""
1269	<	return job requirements to add to jdl files
1269	>	return job requirements to add to jdl files
1270		"""
1271		req = ''
1272		if self.version:
#	Line 1286 | Line 1318 | class Cmssw(JobType):
1318		txt += '       cd $RUNTIME_AREA\n'
1319		txt += '       /bin/rm -rf $WORKING_DIR\n'
1320		txt += '       if [ -d $WORKING_DIR ] ;then\n'
1321	<	txt += '           echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1322	<	txt += '           echo "JOB_EXIT_STATUS = 10017"\n'
1323	<	txt += '           echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n'
1321	>	txt += '           echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1322	>	txt += '           echo "JOB_EXIT_STATUS = 10017"\n'
1323	>	txt += '           echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n'
1324		txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1325		txt += '           rm -f $RUNTIME_AREA/$repo \n'
1326		txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
#	Line 1302 | Line 1334 | class Cmssw(JobType):
1334		txt += '   echo " END SETUP CMS OSG  ENVIRONMENT "\n'
1335
1336		return txt
1337	<
1337	>
1338		### OLI_DANIELE
1339		def wsSetupCMSLCGEnvironment_(self):
1340		"""
#	Line 1351 | Line 1383 | class Cmssw(JobType):
1383		txt += '   echo "### END SETUP CMS LCG ENVIRONMENT ###"\n'
1384		return txt
1385
1386	<	### FEDE FOR DBS OUTPUT PUBLICATION
1386	>	### FEDE FOR DBS OUTPUT PUBLICATION
1387		def modifyReport(self, nj):
1388		"""
1389	<	insert the part of the script that modifies the FrameworkJob Report
1389	>	insert the part of the script that modifies the FrameworkJob Report
1390		"""
1391
1392	<	txt = ''
1361	<	txt += 'echo "Modify Job Report" \n'
1362	<	#txt += 'chmod a+x $RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1363	<	################ FEDE FOR DBS2 #############################################
1364	<	txt += 'chmod a+x $SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1365	<	#############################################################################
1392	>	txt = ''
1393		try:
1394	<	publish_data = int(self.cfg_params['USER.publish_data'])
1394	>	publish_data = int(self.cfg_params['USER.publish_data'])
1395		except KeyError:
1396		publish_data = 0
1397	+	if (publish_data == 1):
1398	+	txt += 'echo "Modify Job Report" \n'
1399	+	#txt += 'chmod a+x $RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1400	+	################ FEDE FOR DBS2 #############################################
1401	+	txt += 'chmod a+x $SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1402	+	#############################################################################
1403	+	#try:
1404	+	#    publish_data = int(self.cfg_params['USER.publish_data'])
1405	+	#except KeyError:
1406	+	#    publish_data = 0
1407
1408	<	txt += 'if [ -z "$SE" ]; then\n'
1409	<	txt += '    SE="" \n'
1410	<	txt += 'fi \n'
1411	<	txt += 'if [ -z "$SE_PATH" ]; then\n'
1412	<	txt += '    SE_PATH="" \n'
1413	<	txt += 'fi \n'
1414	<	txt += 'echo "SE = $SE"\n'
1415	<	txt += 'echo "SE_PATH = $SE_PATH"\n'
1408	>	txt += 'if [ -z "$SE" ]; then\n'
1409	>	txt += '    SE="" \n'
1410	>	txt += 'fi \n'
1411	>	txt += 'if [ -z "$SE_PATH" ]; then\n'
1412	>	txt += '    SE_PATH="" \n'
1413	>	txt += 'fi \n'
1414	>	txt += 'echo "SE = $SE"\n'
1415	>	txt += 'echo "SE_PATH = $SE_PATH"\n'
1416
1417	<	if (publish_data == 1):
1417	>	#if (publish_data == 1):
1418		#processedDataset = self.cfg_params['USER.processed_datasetname']
1419		processedDataset = self.cfg_params['USER.publish_data_name']
1420		txt += 'ProcessedDataset='+processedDataset+'\n'
#	Line 1385 | Line 1422 | class Cmssw(JobType):
1422		txt += 'if [ "$SE_PATH" == "" ]; then\n'
1423		#### FEDE: added slash in LFN ##############
1424		txt += '    FOR_LFN=/copy_problems/ \n'
1425	<	txt += 'else \n'
1425	>	txt += 'else \n'
1426		txt += '    tmp=`echo $SE_PATH | awk -F \'store\' \'{print$2}\'` \n'
1427		#####  FEDE TO BE CHANGED, BECAUSE STORE IS HARDCODED!!!! ########
1428		txt += '    FOR_LFN=/store$tmp \n'
1429	<	txt += 'fi \n'
1429	>	txt += 'fi \n'
1430		txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1431		txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1432		txt += 'echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n'
1433	<	#txt += 'echo "$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1434	<	txt += 'echo "$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1433	>	#txt += 'echo "$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1434	>	txt += 'echo "$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1435		txt += '$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1436		#txt += '$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1437	<
1437	>
1438		txt += 'modifyReport_result=$?\n'
1439		txt += 'echo modifyReport_result = $modifyReport_result\n'
1440		txt += 'if [ $modifyReport_result -ne 0 ]; then\n'
#	Line 1407 | Line 1444 | class Cmssw(JobType):
1444		txt += '    mv NewFrameworkJobReport.xml crab_fjr_$NJob.xml\n'
1445		txt += 'fi\n'
1446		else:
1447	<	txt += 'ProcessedDataset=no_data_to_publish \n'
1447	>	txt += 'echo "no data publication required"\n'
1448	>	#txt += 'ProcessedDataset=no_data_to_publish \n'
1449		#### FEDE: added slash in LFN ##############
1450	<	txt += 'FOR_LFN=/local/ \n'
1451	<	txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1452	<	txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1450	>	#txt += 'FOR_LFN=/local/ \n'
1451	>	#txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1452	>	#txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1453		return txt
1454
1455		def cleanEnv(self):
1456		### OLI_DANIELE
1457	<	txt = ''
1458	<	txt += 'if [ $middleware == OSG ]; then\n'
1457	>	txt = ''
1458	>	txt += 'if [ $middleware == OSG ]; then\n'
1459		txt += '    cd $RUNTIME_AREA\n'
1460		txt += '    echo "Remove working directory: $WORKING_DIR"\n'
1461		txt += '    /bin/rm -rf $WORKING_DIR\n'
1462		txt += '    if [ -d $WORKING_DIR ] ;then\n'
1463	<	txt += '              echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1464	<	txt += '              echo "JOB_EXIT_STATUS = 60999"\n'
1465	<	txt += '              echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
1466	<	txt += '              dumpStatus $RUNTIME_AREA/$repo\n'
1463	>	txt += '              echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1464	>	txt += '              echo "JOB_EXIT_STATUS = 60999"\n'
1465	>	txt += '              echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
1466	>	txt += '              dumpStatus $RUNTIME_AREA/$repo\n'
1467		txt += '        rm -f $RUNTIME_AREA/$repo \n'
1468		txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1469		txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
#	Line 1442 | Line 1480 | class Cmssw(JobType):
1480
1481		def setTaskid_(self):
1482		self._taskId = self.cfg_params['taskId']
1483	<
1483	>
1484		def getTaskid(self):
1485		return self._taskId
1486
#	Line 1454 | Line 1492 | class Cmssw(JobType):
1492		for e in old:
1493		nd[e]=0
1494		return nd.keys()
1495	+
1496	+
1497	+	def checkOut(self, limit):
1498	+	"""
1499	+	check the dimension of the output files
1500	+	"""
1501	+	txt = 'echo "*****************************************"\n'
1502	+	txt += 'echo "** Starting output sandbox limit check **"\n'
1503	+	txt += 'echo "*****************************************"\n'
1504	+	allOutFiles = ""
1505	+	listOutFiles = []
1506	+	for fileOut in (self.output_file+self.output_file_sandbox):
1507	+	if fileOut.find('crab_fjr') == -1:
1508	+	allOutFiles = allOutFiles + " " + self.numberFile_(fileOut, '$NJob')
1509	+	listOutFiles.append(self.numberFile_(fileOut, '$NJob'))
1510	+	txt += 'echo "OUTPUT files: '+str(allOutFiles)+'";\n'
1511	+	txt += 'ls -gGhrta;\n'
1512	+	txt += 'sum=0;\n'
1513	+	txt += 'for file in '+str(allOutFiles)+' ; do\n'
1514	+	txt += '    if [ -e $file ]; then\n'
1515	+	txt += '        tt=`ls -gGrta $file | awk \'{ print $3 }\'`\n'
1516	+	txt += '        sum=`expr $sum + $tt`\n'
1517	+	txt += '    else\n'
1518	+	txt += '        echo "WARNING: output file $file not found!"\n'
1519	+	txt += '    fi\n'
1520	+	txt += 'done\n'
1521	+	txt += 'echo "Total Output dimension: $sum";\n'
1522	+	txt += 'limit='+str(limit)+';\n'
1523	+	txt += 'echo "OUTPUT FILES LIMIT SET TO: $limit";\n'
1524	+	txt += 'if [ $limit -lt $sum ]; then\n'
1525	+	txt += '    echo "WARNING: output files have to big size - something will be lost;"\n'
1526	+	txt += '    echo "         checking the output file sizes..."\n'
1527	+	"""
1528	+	txt += '    dim=0;\n'
1529	+	txt += '    exclude=0;\n'
1530	+	txt += '    for files in '+str(allOutFiles)+' ; do\n'
1531	+	txt += '        sumTemp=0;\n'
1532	+	txt += '        for file2 in '+str(allOutFiles)+' ; do\n'
1533	+	txt += '            if [ $file != $file2 ]; then\n'
1534	+	txt += '                tt=`ls -gGrta $file2 | awk \'{ print $3 }\';`\n'
1535	+	txt += '                sumTemp=`expr $sumTemp + $tt`;\n'
1536	+	txt += '            fi\n'
1537	+	txt += '        done\n'
1538	+	txt += '        if [ $sumTemp -lt $limit ]; then\n'
1539	+	txt += '            if [ $dim -lt $sumTemp ]; then\n'
1540	+	txt += '                dim=$sumTemp;\n'
1541	+	txt += '                exclude=$file;\n'
1542	+	txt += '            fi\n'
1543	+	txt += '        fi\n'
1544	+	txt += '    done\n'
1545	+	txt += '    echo "Dimension calculated: $dim"; echo "File to exclude: $exclude";\n'
1546	+	"""
1547	+	txt += '    tot=0;\n'
1548	+	txt += '    for file2 in '+str(allOutFiles)+' ; do\n'
1549	+	txt += '        tt=`ls -gGrta $file2 | awk \'{ print $3 }\';`\n'
1550	+	txt += '        tot=`expr $tot + $tt`;\n'
1551	+	txt += '        if [ $limit -lt $tot ]; then\n'
1552	+	txt += '            tot=`expr $tot - $tt`;\n'
1553	+	txt += '            fileLast=$file;\n'
1554	+	txt += '            break;\n'
1555	+	txt += '        fi\n'
1556	+	txt += '    done\n'
1557	+	txt += '    echo "Dimension calculated: $tot"; echo "First file to exclude: $file";\n'
1558	+	txt += '    flag=0;\n'
1559	+	txt += '    for filess in '+str(allOutFiles)+' ; do\n'
1560	+	txt += '        if [ $fileLast = $filess ]; then\n'
1561	+	txt += '            flag=1;\n'
1562	+	txt += '        fi\n'
1563	+	txt += '        if [ $flag -eq 1 ]; then\n'
1564	+	txt += '            rm -f $filess;\n'
1565	+	txt += '        fi\n'
1566	+	txt += '    done\n'
1567	+	txt += '    ls -agGhrt;\n'
1568	+	txt += '    echo "WARNING: output files are too big in dimension: can not put in the output_sandbox.";\n'
1569	+	txt += '    echo "JOB_EXIT_STATUS = 70000";\n'
1570	+	txt += '    exit_status=70000;\n'
1571	+	txt += 'else'
1572	+	txt += '    echo "Total Output dimension $sum is fine.";\n'
1573	+	txt += 'fi\n'
1574	+	txt += 'echo "*****************************************"\n'
1575	+	txt += 'echo "*** Ending output sandbox limit check ***"\n'
1576	+	txt += 'echo "*****************************************"\n'
1577	+	return txt

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines