ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/cms_cmssw.py

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.49 by slacapra, Thu Oct 5 15:32:20 2006 UTC vs.
Revision 1.117 by fanzago, Fri Aug 17 10:45:26 2007 UTC

#	Line 2 | Line 2 | from JobType import JobType
2		from crab_logger import Logger
3		from crab_exceptions import *
4		from crab_util import *
5	<	import math
5	>	from BlackWhiteListParser import BlackWhiteListParser
6		import common
7	–	import PsetManipulator
8	–
9	–	import DBSInfo
10	–	import DataDiscovery
11	–	import DataLocation
7		import Scram
8
9	<	import glob, os, string, re
9	>	import os, string, glob
10
11		class Cmssw(JobType):
12		def __init__(self, cfg_params, ncjobs):
13		JobType.__init__(self, 'CMSSW')
14		common.logger.debug(3,'CMSSW::__init__')
15
21	–	# Marco.
16		self._params = {}
17		self.cfg_params = cfg_params
18
19	+	# init BlackWhiteListParser
20	+	self.blackWhiteListParser = BlackWhiteListParser(cfg_params)
21	+
22	+	try:
23	+	self.MaxTarBallSize = float(self.cfg_params['EDG.maxtarballsize'])
24	+	except KeyError:
25	+	self.MaxTarBallSize = 9.5
26	+
27		# number of jobs requested to be created, limit obj splitting
28		self.ncjobs = ncjobs
29
30		log = common.logger
31
32		self.scram = Scram.Scram(cfg_params)
31	–	scramArea = ''
33		self.additional_inbox_files = []
34		self.scriptExe = ''
35		self.executable = ''
36	+	self.executable_arch = self.scram.getArch()
37		self.tgz_name = 'default.tgz'
38	+	self.additional_tgz_name = 'additional.tgz'
39	+	self.scriptName = 'CMSSW.sh'
40		self.pset = ''      #scrip use case Da
41		self.datasetPath = '' #scrip use case Da
42
43	+	# set FJR file name
44	+	self.fjrFileName = 'crab_fjr.xml'
45	+
46		self.version = self.scram.getSWVersion()
47	+
48	+	#
49	+	# Try to block creation in case of arch/version mismatch
50	+	#
51	+
52	+	a = string.split(self.version, "_")
53	+
54	+	if int(a[1]) == 1 and (int(a[2]) < 5 and self.executable_arch.find('slc4') == 0):
55	+	msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch)
56	+	raise CrabException(msg)
57	+	if int(a[1]) == 1 and (int(a[2]) >= 5 and self.executable_arch.find('slc3') == 0):
58	+	msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch)
59	+	raise CrabException(msg)
60	+
61	+	common.taskDB.setDict('codeVersion',self.version)
62		self.setParam_('application', self.version)
63
64		### collect Data cards
65	+
66	+	## get DBS mode
67	+	try:
68	+	self.use_dbs_1 = int(self.cfg_params['CMSSW.use_dbs_1'])
69	+	except KeyError:
70	+	self.use_dbs_1 = 0
71	+
72		try:
73		tmp =  cfg_params['CMSSW.datasetpath']
74		log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
#	Line 59 | Line 88 | class Cmssw(JobType):
88		self.setParam_('dataset', 'None')
89		self.setParam_('owner', 'None')
90		else:
91	<	datasetpath_split = self.datasetPath.split("/")
92	<	self.setParam_('dataset', datasetpath_split[1])
93	<	self.setParam_('owner', datasetpath_split[-1])
94	<
91	>	try:
92	>	datasetpath_split = self.datasetPath.split("/")
93	>	# standard style
94	>	if self.use_dbs_1 == 1 :
95	>	self.setParam_('dataset', datasetpath_split[1])
96	>	self.setParam_('owner', datasetpath_split[-1])
97	>	else:
98	>	self.setParam_('dataset', datasetpath_split[1])
99	>	self.setParam_('owner', datasetpath_split[2])
100	>	except:
101	>	self.setParam_('dataset', self.datasetPath)
102	>	self.setParam_('owner', self.datasetPath)
103	>
104		self.setTaskid_()
105		self.setParam_('taskId', self.cfg_params['taskId'])
106
#	Line 94 | Line 132 | class Cmssw(JobType):
132		raise CrabException("PSet file missing. Cannot run cmsRun ")
133
134		# output files
135	+	## stuff which must be returned always via sandbox
136	+	self.output_file_sandbox = []
137	+
138	+	# add fjr report by default via sandbox
139	+	self.output_file_sandbox.append(self.fjrFileName)
140	+
141	+	# other output files to be returned via sandbox or copied to SE
142		try:
143		self.output_file = []
99	–
144		tmp = cfg_params['CMSSW.output_file']
145		if tmp != '':
146		tmpOutFiles = string.split(cfg_params['CMSSW.output_file'],',')
#	Line 106 | Line 150 | class Cmssw(JobType):
150		self.output_file.append(tmp)
151		pass
152		else:
153	<	log.message("No output file defined: only stdout/err will be available")
153	>	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
154		pass
155		pass
156		except KeyError:
157	<	log.message("No output file defined: only stdout/err will be available")
157	>	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
158		pass
159
160		# script_exe file as additional file in inputSandbox
#	Line 118 | Line 162 | class Cmssw(JobType):
162		self.scriptExe = cfg_params['USER.script_exe']
163		if self.scriptExe != '':
164		if not os.path.isfile(self.scriptExe):
165	<	msg ="WARNING. file "+self.scriptExe+" not found"
165	>	msg ="ERROR. file "+self.scriptExe+" not found"
166		raise CrabException(msg)
167		self.additional_inbox_files.append(string.strip(self.scriptExe))
168		except KeyError:
169		self.scriptExe = ''
170	+
171		#CarlosDaniele
172		if self.datasetPath == None and self.pset == None and self.scriptExe == '' :
173	<	msg ="WARNING. script_exe  not defined"
173	>	msg ="Error. script_exe  not defined"
174		raise CrabException(msg)
175
176		## additional input files
177		try:
178		tmpAddFiles = string.split(cfg_params['USER.additional_input_files'],',')
179		for tmp in tmpAddFiles:
180	+	tmp = string.strip(tmp)
181		dirname = ''
182		if not tmp[0]=="/": dirname = "."
183	<	files = glob.glob(os.path.join(dirname, tmp))
183	>	files = []
184	>	if string.find(tmp,"*")>-1:
185	>	files = glob.glob(os.path.join(dirname, tmp))
186	>	if len(files)==0:
187	>	raise CrabException("No additional input file found with this pattern: "+tmp)
188	>	else:
189	>	files.append(tmp)
190		for file in files:
191		if not os.path.exists(file):
192		raise CrabException("Additional input file not found: "+file)
193		pass
194	+	# fname = string.split(file, '/')[-1]
195	+	# storedFile = common.work_space.pathForTgz()+'share/'+fname
196	+	# shutil.copyfile(file, storedFile)
197		self.additional_inbox_files.append(string.strip(file))
198		pass
199		pass
#	Line 197 | Line 252 | class Cmssw(JobType):
252		except KeyError:
253		self.sourceSeedVtx = None
254		common.logger.debug(5,"No vertex seed given")
255	+
256	+	try:
257	+	self.sourceSeedG4 = int(cfg_params['CMSSW.g4_seed'])
258	+	except KeyError:
259	+	self.sourceSeedG4 = None
260	+	common.logger.debug(5,"No g4 sim hits seed given")
261	+
262	+	try:
263	+	self.sourceSeedMix = int(cfg_params['CMSSW.mix_seed'])
264	+	except KeyError:
265	+	self.sourceSeedMix = None
266	+	common.logger.debug(5,"No mix seed given")
267	+
268	+	try:
269	+	self.firstRun = int(cfg_params['CMSSW.first_run'])
270	+	except KeyError:
271	+	self.firstRun = None
272	+	common.logger.debug(5,"No first run given")
273		if self.pset != None: #CarlosDaniele
274	<	self.PsetEdit = PsetManipulator.PsetManipulator(self.pset) #Daniele Pset
274	>	ver = string.split(self.version,"_")
275	>	if (int(ver[1])>=1 and int(ver[2])>=5):
276	>	import PsetManipulator150 as pp
277	>	else:
278	>	import PsetManipulator as pp
279	>	PsetEdit = pp.PsetManipulator(self.pset) #Daniele Pset
280
281		#DBSDLS-start
282		## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
#	Line 220 | Line 298 | class Cmssw(JobType):
298		self.jobSplittingForScript()
299		else:
300		self.jobSplittingNoInput()
301	<	else: self.jobSplittingByBlocks(blockSites)
301	>	else:
302	>	self.jobSplittingByBlocks(blockSites)
303
304		# modify Pset
305		if self.pset != None: #CarlosDaniele
306		try:
307		if (self.datasetPath): # standard job
308		# allow to processa a fraction of events in a file
309	<	self.PsetEdit.inputModule("INPUT")
310	<	self.PsetEdit.maxEvent("INPUTMAXEVENTS")
311	<	self.PsetEdit.skipEvent("INPUTSKIPEVENTS")
309	>	PsetEdit.inputModule("INPUT")
310	>	PsetEdit.maxEvent("INPUTMAXEVENTS")
311	>	PsetEdit.skipEvent("INPUTSKIPEVENTS")
312		else:  # pythia like job
313	<	self.PsetEdit.maxEvent(self.eventsPerJob)
313	>	PsetEdit.maxEvent(self.eventsPerJob)
314	>	if (self.firstRun):
315	>	PsetEdit.pythiaFirstRun("INPUTFIRSTRUN")  #First Run
316		if (self.sourceSeed) :
317	<	self.PsetEdit.pythiaSeed("INPUT")
317	>	PsetEdit.pythiaSeed("INPUT")
318		if (self.sourceSeedVtx) :
319	<	self.PsetEdit.pythiaSeedVtx("INPUTVTX")
320	<	self.PsetEdit.psetWriter(self.configFilename())
319	>	PsetEdit.vtxSeed("INPUTVTX")
320	>	if (self.sourceSeedG4) :
321	>	self.PsetEdit.g4Seed("INPUTG4")
322	>	if (self.sourceSeedMix) :
323	>	self.PsetEdit.mixSeed("INPUTMIX")
324	>	# add FrameworkJobReport to parameter-set
325	>	PsetEdit.addCrabFJR(self.fjrFileName)
326	>	PsetEdit.psetWriter(self.configFilename())
327		except:
328		msg='Error while manipuliating ParameterSet: exiting...'
329		raise CrabException(msg)
330
331		def DataDiscoveryAndLocation(self, cfg_params):
332
333	+	import DataDiscovery
334	+	import DataDiscovery_DBS2
335	+	import DataLocation
336		common.logger.debug(10,"CMSSW::DataDiscoveryAndLocation()")
337
338		datasetPath=self.datasetPath
339
250	–	## TODO
251	–	dataTiersList = ""
252	–	dataTiers = dataTiersList.split(',')
253	–
340		## Contact the DBS
341	<	common.logger.message("Contacting DBS...")
341	>	common.logger.message("Contacting Data Discovery Services ...")
342		try:
343	<	self.pubdata=DataDiscovery.DataDiscovery(datasetPath, dataTiers, cfg_params)
343	>
344	>	if self.use_dbs_1 == 1 :
345	>	self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params)
346	>	else :
347	>	self.pubdata=DataDiscovery_DBS2.DataDiscovery_DBS2(datasetPath, cfg_params)
348		self.pubdata.fetchDBSInfo()
349
350		except DataDiscovery.NotExistingDatasetError, ex :
351		msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
352		raise CrabException(msg)
263	–
353		except DataDiscovery.NoDataTierinProvenanceError, ex :
354		msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
355		raise CrabException(msg)
356		except DataDiscovery.DataDiscoveryError, ex:
357	<	msg = 'ERROR ***: failed Data Discovery in DBS  %s'%ex.getErrorMessage()
357	>	msg = 'ERROR ***: failed Data Discovery in DBS :  %s'%ex.getErrorMessage()
358	>	raise CrabException(msg)
359	>	except DataDiscovery_DBS2.NotExistingDatasetError_DBS2, ex :
360	>	msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
361	>	raise CrabException(msg)
362	>	except DataDiscovery_DBS2.NoDataTierinProvenanceError_DBS2, ex :
363	>	msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
364	>	raise CrabException(msg)
365	>	except DataDiscovery_DBS2.DataDiscoveryError_DBS2, ex:
366	>	msg = 'ERROR ***: failed Data Discovery in DBS :  %s'%ex.getErrorMessage()
367		raise CrabException(msg)
270	–
271	–	## get list of all required data in the form of dbs paths  (dbs path = /dataset/datatier/owner)
272	–	## self.DBSPaths=self.pubdata.getDBSPaths()
273	–	common.logger.message("Required data are :"+self.datasetPath)
368
369		self.filesbyblock=self.pubdata.getFiles()
370		self.eventsbyblock=self.pubdata.getEventsPerBlock()
371		self.eventsbyfile=self.pubdata.getEventsPerFile()
278	–	# print str(self.filesbyblock)
279	–	# print 'self.eventsbyfile',len(self.eventsbyfile)
280	–	# print str(self.eventsbyfile)
372
373		## get max number of events
374		self.maxEvents=self.pubdata.getMaxEvents() ##  self.maxEvents used in Creator.py
284	–	common.logger.message("The number of available events is %s\n"%self.maxEvents)
375
286	–	common.logger.message("Contacting DLS...")
376		## Contact the DLS and build a list of sites hosting the fileblocks
377		try:
378		dataloc=DataLocation.DataLocation(self.filesbyblock.keys(),cfg_params)
#	Line 296 | Line 385 | class Cmssw(JobType):
385		sites = dataloc.getSites()
386		allSites = []
387		listSites = sites.values()
388	<	for list in listSites:
389	<	for oneSite in list:
388	>	for listSite in listSites:
389	>	for oneSite in listSite:
390		allSites.append(oneSite)
391		allSites = self.uniquelist(allSites)
392
393	<	common.logger.message("Sites ("+str(len(allSites))+") hosting part/all of dataset: "+str(allSites))
394	<	common.logger.debug(6, "List of Sites: "+str(allSites))
393	>	# screen output
394	>	common.logger.message("Requested dataset: " + datasetPath + " has " + str(self.maxEvents) + " events in " + str(len(self.filesbyblock.keys())) + " blocks.\n")
395	>
396		return sites
397
398		def jobSplittingByBlocks(self, blockSites):
#	Line 364 | Line 454 | class Cmssw(JobType):
454		jobCount = 0
455		list_of_lists = []
456
457	+	# list tracking which jobs are in which jobs belong to which block
458	+	jobsOfBlock = {}
459	+
460		# ---- Iterate over the blocks in the dataset until ---- #
461		# ---- we've met the requested total # of events    ---- #
462		while ( (eventsRemaining > 0) and (blockCount < numBlocksInDataset) and (jobCount < totalNumberOfJobs)):
463		block = blocks[blockCount]
464		blockCount += 1
465	+	if block not in jobsOfBlock.keys() :
466	+	jobsOfBlock[block] = []
467
468	<
469	<	numEventsInBlock = self.eventsbyblock[block]
470	<	common.logger.debug(5,'Events in Block File '+str(numEventsInBlock))
468	>	if self.eventsbyblock.has_key(block) :
469	>	numEventsInBlock = self.eventsbyblock[block]
470	>	common.logger.debug(5,'Events in Block File '+str(numEventsInBlock))
471
472	<	files = self.filesbyblock[block]
473	<	numFilesInBlock = len(files)
474	<	if (numFilesInBlock <= 0):
475	<	continue
476	<	fileCount = 0
477	<
478	<	# ---- New block => New job ---- #
479	<	parString = "\\{"
480	<	# counter for number of events in files currently worked on
481	<	filesEventCount = 0
482	<	# flag if next while loop should touch new file
483	<	newFile = 1
484	<	# job event counter
485	<	jobSkipEventCount = 0
472	>	files = self.filesbyblock[block]
473	>	numFilesInBlock = len(files)
474	>	if (numFilesInBlock <= 0):
475	>	continue
476	>	fileCount = 0
477	>
478	>	# ---- New block => New job ---- #
479	>	parString = "\\{"
480	>	# counter for number of events in files currently worked on
481	>	filesEventCount = 0
482	>	# flag if next while loop should touch new file
483	>	newFile = 1
484	>	# job event counter
485	>	jobSkipEventCount = 0
486
487	<	# ---- Iterate over the files in the block until we've met the requested ---- #
488	<	# ---- total # of events or we've gone over all the files in this block  ---- #
489	<	while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) and (jobCount < totalNumberOfJobs) ):
490	<	file = files[fileCount]
491	<	if newFile :
492	<	try:
493	<	numEventsInFile = self.eventsbyfile[file]
494	<	common.logger.debug(6, "File "+str(file)+" has "+str(numEventsInFile)+" events")
495	<	# increase filesEventCount
496	<	filesEventCount += numEventsInFile
497	<	# Add file to current job
498	<	parString += '\\\"' + file + '\\\"\,'
499	<	newFile = 0
500	<	except KeyError:
501	<	common.logger.message("File "+str(file)+" has unknown number of events: skipping")
487	>	# ---- Iterate over the files in the block until we've met the requested ---- #
488	>	# ---- total # of events or we've gone over all the files in this block  ---- #
489	>	while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) and (jobCount < totalNumberOfJobs) ):
490	>	file = files[fileCount]
491	>	if newFile :
492	>	try:
493	>	numEventsInFile = self.eventsbyfile[file]
494	>	common.logger.debug(6, "File "+str(file)+" has "+str(numEventsInFile)+" events")
495	>	# increase filesEventCount
496	>	filesEventCount += numEventsInFile
497	>	# Add file to current job
498	>	parString += '\\\"' + file + '\\\"\,'
499	>	newFile = 0
500	>	except KeyError:
501	>	common.logger.message("File "+str(file)+" has unknown number of events: skipping")
502
503
504	<	# if less events in file remain than eventsPerJobRequested
505	<	if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested ) :
506	<	# if last file in block
507	<	if ( fileCount == numFilesInBlock-1 ) :
508	<	# end job using last file, use remaining events in block
504	>	# if less events in file remain than eventsPerJobRequested
505	>	if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested ) :
506	>	# if last file in block
507	>	if ( fileCount == numFilesInBlock-1 ) :
508	>	# end job using last file, use remaining events in block
509	>	# close job and touch new file
510	>	fullString = parString[:-2]
511	>	fullString += '\\}'
512	>	list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
513	>	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(filesEventCount - jobSkipEventCount)+" events (last file in block).")
514	>	self.jobDestination.append(blockSites[block])
515	>	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
516	>	# fill jobs of block dictionary
517	>	jobsOfBlock[block].append(jobCount+1)
518	>	# reset counter
519	>	jobCount = jobCount + 1
520	>	totalEventCount = totalEventCount + filesEventCount - jobSkipEventCount
521	>	eventsRemaining = eventsRemaining - filesEventCount + jobSkipEventCount
522	>	jobSkipEventCount = 0
523	>	# reset file
524	>	parString = "\\{"
525	>	filesEventCount = 0
526	>	newFile = 1
527	>	fileCount += 1
528	>	else :
529	>	# go to next file
530	>	newFile = 1
531	>	fileCount += 1
532	>	# if events in file equal to eventsPerJobRequested
533	>	elif ( filesEventCount - jobSkipEventCount == eventsPerJobRequested ) :
534		# close job and touch new file
535		fullString = parString[:-2]
536		fullString += '\\}'
537	<	list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
538	<	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(filesEventCount - jobSkipEventCount)+" events (last file in block).")
537	>	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
538	>	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
539		self.jobDestination.append(blockSites[block])
540		common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
541	+	jobsOfBlock[block].append(jobCount+1)
542		# reset counter
543		jobCount = jobCount + 1
544	<	totalEventCount = totalEventCount + filesEventCount - jobSkipEventCount
545	<	eventsRemaining = eventsRemaining - filesEventCount + jobSkipEventCount
544	>	totalEventCount = totalEventCount + eventsPerJobRequested
545	>	eventsRemaining = eventsRemaining - eventsPerJobRequested
546		jobSkipEventCount = 0
547		# reset file
548		parString = "\\{"
549		filesEventCount = 0
550		newFile = 1
551		fileCount += 1
552	+
553	+	# if more events in file remain than eventsPerJobRequested
554		else :
555	<	# go to next file
556	<	newFile = 1
557	<	fileCount += 1
558	<	# if events in file equal to eventsPerJobRequested
559	<	elif ( filesEventCount - jobSkipEventCount == eventsPerJobRequested ) :
560	<	# close job and touch new file
561	<	fullString = parString[:-2]
562	<	fullString += '\\}'
563	<	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
564	<	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
565	<	self.jobDestination.append(blockSites[block])
566	<	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
567	<	# reset counter
568	<	jobCount = jobCount + 1
569	<	totalEventCount = totalEventCount + eventsPerJobRequested
570	<	eventsRemaining = eventsRemaining - eventsPerJobRequested
571	<	jobSkipEventCount = 0
572	<	# reset file
573	<	parString = "\\{"
574	<	filesEventCount = 0
575	<	newFile = 1
453	<	fileCount += 1
454	<
455	<	# if more events in file remain than eventsPerJobRequested
456	<	else :
457	<	# close job but don't touch new file
458	<	fullString = parString[:-2]
459	<	fullString += '\\}'
460	<	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
461	<	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
462	<	self.jobDestination.append(blockSites[block])
463	<	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
464	<	# increase counter
465	<	jobCount = jobCount + 1
466	<	totalEventCount = totalEventCount + eventsPerJobRequested
467	<	eventsRemaining = eventsRemaining - eventsPerJobRequested
468	<	# calculate skip events for last file
469	<	# use filesEventCount (contains several files), jobSkipEventCount and eventsPerJobRequest
470	<	jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file])
471	<	# remove all but the last file
472	<	filesEventCount = self.eventsbyfile[file]
473	<	parString = "\\{"
474	<	parString += '\\\"' + file + '\\\"\,'
475	<	pass # END if
476	<	pass # END while (iterate over files in the block)
555	>	# close job but don't touch new file
556	>	fullString = parString[:-2]
557	>	fullString += '\\}'
558	>	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
559	>	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
560	>	self.jobDestination.append(blockSites[block])
561	>	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
562	>	jobsOfBlock[block].append(jobCount+1)
563	>	# increase counter
564	>	jobCount = jobCount + 1
565	>	totalEventCount = totalEventCount + eventsPerJobRequested
566	>	eventsRemaining = eventsRemaining - eventsPerJobRequested
567	>	# calculate skip events for last file
568	>	# use filesEventCount (contains several files), jobSkipEventCount and eventsPerJobRequest
569	>	jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file])
570	>	# remove all but the last file
571	>	filesEventCount = self.eventsbyfile[file]
572	>	parString = "\\{"
573	>	parString += '\\\"' + file + '\\\"\,'
574	>	pass # END if
575	>	pass # END while (iterate over files in the block)
576		pass # END while (iterate over blocks in the dataset)
577		self.ncjobs = self.total_number_of_jobs = jobCount
578		if (eventsRemaining > 0 and jobCount < totalNumberOfJobs ):
579		common.logger.message("Could not run on all requested events because some blocks not hosted at allowed sites.")
580	<	common.logger.message("\n"+str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n")
580	>	common.logger.message(str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n")
581
582	+	# screen output
583	+	screenOutput = "List of jobs and available destination sites:\n\n"
584	+
585	+	blockCounter = 0
586	+	for block in blocks:
587	+	if block in jobsOfBlock.keys() :
588	+	blockCounter += 1
589	+	screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),','.join(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)))
590	+
591	+	common.logger.message(screenOutput)
592	+
593		self.list_of_args = list_of_lists
594		return
595
#	Line 497 | Line 607 | class Cmssw(JobType):
607		raise CrabException(msg)
608
609		if (self.selectEventsPerJob):
610	<	self.total_number_of_jobs = int(self.total_number_of_events/self.eventsPerJob)
610	>	if (self.selectTotalNumberEvents):
611	>	self.total_number_of_jobs = int(self.total_number_of_events/self.eventsPerJob)
612	>	elif(self.selectNumberOfJobs) :
613	>	self.total_number_of_jobs =self.theNumberOfJobs
614	>	self.total_number_of_events =int(self.theNumberOfJobs*self.eventsPerJob)
615	>
616		elif (self.selectNumberOfJobs) :
617		self.total_number_of_jobs = self.theNumberOfJobs
618		self.eventsPerJob = int(self.total_number_of_events/self.total_number_of_jobs)
619	<
619	>
620		common.logger.debug(5,'N jobs  '+str(self.total_number_of_jobs))
621
622		# is there any remainder?
#	Line 519 | Line 634 | class Cmssw(JobType):
634		## Since there is no input, any site is good
635		# self.jobDestination.append(["Any"])
636		self.jobDestination.append([""]) #must be empty to write correctly the xml
637	+	args=[]
638	+	if (self.firstRun):
639	+	## pythia first run
640	+	#self.list_of_args.append([(str(self.firstRun)+str(i))])
641	+	args.append(str(self.firstRun)+str(i))
642	+	else:
643	+	## no first run
644	+	#self.list_of_args.append([str(i)])
645	+	args.append(str(i))
646		if (self.sourceSeed):
647	+	args.append(str(self.sourceSeed)+str(i))
648		if (self.sourceSeedVtx):
649	<	## pythia + vtx random seed
650	<	self.list_of_args.append([
651	<	str(self.sourceSeed)+str(i),
652	<	str(self.sourceSeedVtx)+str(i)
653	<	])
654	<	else:
655	<	## only pythia random seed
656	<	self.list_of_args.append([(str(self.sourceSeed)+str(i))])
657	<	else:
658	<	## no random seed
659	<	self.list_of_args.append([str(i)])
660	<	#print self.list_of_args
649	>	## + vtx random seed
650	>	args.append(str(self.sourceSeedVtx)+str(i))
651	>	if (self.sourceSeedG4):
652	>	## + G4 random seed
653	>	args.append(str(self.sourceSeedG4)+str(i))
654	>	if (self.sourceSeedMix):
655	>	## + Mix random seed
656	>	args.append(str(self.sourceSeedMix)+str(i))
657	>	pass
658	>	pass
659	>	self.list_of_args.append(args)
660	>	pass
661	>
662	>	# print self.list_of_args
663
664		return
665
#	Line 597 | Line 724 | class Cmssw(JobType):
724		"""
725
726		# if it exist, just return it
727	<	self.tgzNameWithPath = common.work_space.shareDir()+self.tgz_name
727	>	#
728	>	# Marco. Let's start to use relative path for Boss XML files
729	>	#
730	>	self.tgzNameWithPath = common.work_space.pathForTgz()+'share/'+self.tgz_name
731		if os.path.exists(self.tgzNameWithPath):
732		return self.tgzNameWithPath
733
#	Line 611 | Line 741 | class Cmssw(JobType):
741		# First of all declare the user Scram area
742		swArea = self.scram.getSWArea_()
743		#print "swArea = ", swArea
744	<	swVersion = self.scram.getSWVersion()
745	<	#print "swVersion = ", swVersion
744	>	# swVersion = self.scram.getSWVersion()
745	>	# print "swVersion = ", swVersion
746		swReleaseTop = self.scram.getReleaseTop_()
747		#print "swReleaseTop = ", swReleaseTop
748
#	Line 620 | Line 750 | class Cmssw(JobType):
750		if swReleaseTop == '' or swArea == swReleaseTop:
751		return
752
753	<	filesToBeTarred = []
754	<	## First find the executable
755	<	if (self.executable != ''):
756	<	exeWithPath = self.scram.findFile_(executable)
757	<	#           print exeWithPath
758	<	if ( not exeWithPath ):
759	<	raise CrabException('User executable '+executable+' not found')
760	<
761	<	## then check if it's private or not
762	<	if exeWithPath.find(swReleaseTop) == -1:
763	<	# the exe is private, so we must ship
764	<	common.logger.debug(5,"Exe "+exeWithPath+" to be tarred")
765	<	path = swArea+'/'
766	<	exe = string.replace(exeWithPath, path,'')
767	<	filesToBeTarred.append(exe)
768	<	pass
769	<	else:
770	<	# the exe is from release, we'll find it on WN
771	<	pass
772	<
773	<	## Now get the libraries: only those in local working area
774	<	libDir = 'lib'
775	<	lib = swArea+'/' +libDir
776	<	common.logger.debug(5,"lib "+lib+" to be tarred")
777	<	if os.path.exists(lib):
778	<	filesToBeTarred.append(libDir)
779	<
780	<	## Now check if module dir is present
781	<	moduleDir = 'module'
782	<	if os.path.isdir(swArea+'/'+moduleDir):
783	<	filesToBeTarred.append(moduleDir)
784	<
785	<	## Now check if the Data dir is present
786	<	dataDir = 'src/Data/'
787	<	if os.path.isdir(swArea+'/'+dataDir):
788	<	filesToBeTarred.append(dataDir)
789	<
790	<	## Create the tar-ball
791	<	if len(filesToBeTarred)>0:
792	<	cwd = os.getcwd()
793	<	os.chdir(swArea)
794	<	tarcmd = 'tar zcvf ' + self.tgzNameWithPath + ' '
795	<	for line in filesToBeTarred:
796	<	tarcmd = tarcmd + line + ' '
797	<	cout = runCommand(tarcmd)
798	<	if not cout:
799	<	raise CrabException('Could not create tar-ball')
800	<	os.chdir(cwd)
801	<	else:
802	<	common.logger.debug(5,"No files to be to be tarred")
753	>	import tarfile
754	>	try: # create tar ball
755	>	tar = tarfile.open(self.tgzNameWithPath, "w:gz")
756	>	## First find the executable
757	>	if (self.executable != ''):
758	>	exeWithPath = self.scram.findFile_(executable)
759	>	if ( not exeWithPath ):
760	>	raise CrabException('User executable '+executable+' not found')
761	>
762	>	## then check if it's private or not
763	>	if exeWithPath.find(swReleaseTop) == -1:
764	>	# the exe is private, so we must ship
765	>	common.logger.debug(5,"Exe "+exeWithPath+" to be tarred")
766	>	path = swArea+'/'
767	>	# distinguish case when script is in user project area or given by full path somewhere else
768	>	if exeWithPath.find(path) >= 0 :
769	>	exe = string.replace(exeWithPath, path,'')
770	>	tar.add(path+exe,os.path.basename(executable))
771	>	else :
772	>	tar.add(exeWithPath,os.path.basename(executable))
773	>	pass
774	>	else:
775	>	# the exe is from release, we'll find it on WN
776	>	pass
777	>
778	>	## Now get the libraries: only those in local working area
779	>	libDir = 'lib'
780	>	lib = swArea+'/' +libDir
781	>	common.logger.debug(5,"lib "+lib+" to be tarred")
782	>	if os.path.exists(lib):
783	>	tar.add(lib,libDir)
784	>
785	>	## Now check if module dir is present
786	>	moduleDir = 'module'
787	>	module = swArea + '/' + moduleDir
788	>	if os.path.isdir(module):
789	>	tar.add(module,moduleDir)
790	>
791	>	## Now check if any data dir(s) is present
792	>	swAreaLen=len(swArea)
793	>	for root, dirs, files in os.walk(swArea):
794	>	if "data" in dirs:
795	>	common.logger.debug(5,"data "+root+"/data"+" to be tarred")
796	>	tar.add(root+"/data",root[swAreaLen:]+"/data")
797	>
798	>	## Add ProdAgent dir to tar
799	>	paDir = 'ProdAgentApi'
800	>	pa = os.environ['CRABDIR'] + '/' + 'ProdAgentApi'
801	>	if os.path.isdir(pa):
802	>	tar.add(pa,paDir)
803	>
804	>	### FEDE FOR DBS PUBLICATION
805	>	## Add PRODCOMMON dir to tar
806	>	prodcommonDir = 'ProdCommon'
807	>	prodcommonPath = os.environ['CRABDIR'] + '/' + 'ProdCommon'
808	>	if os.path.isdir(prodcommonPath):
809	>	tar.add(prodcommonPath,prodcommonDir)
810	>	#############################
811	>
812	>	common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
813	>	tar.close()
814	>	except :
815	>	raise CrabException('Could not create tar-ball')
816	>
817	>	## check for tarball size
818	>	tarballinfo = os.stat(self.tgzNameWithPath)
819	>	if ( tarballinfo.st_size > self.MaxTarBallSize*1024*1024 ) :
820	>	raise CrabException('Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' + str(self.MaxTarBallSize) + ' MB input sandbox limit and not supported by the used GRID submission system. Please make sure that no unnecessary files are in all data directories in your local CMSSW project area as they are automatically packed into the input sandbox.')
821	>
822	>	## create tar-ball with ML stuff
823	>	self.MLtgzfile =  common.work_space.pathForTgz()+'share/MLfiles.tgz'
824	>	try:
825	>	tar = tarfile.open(self.MLtgzfile, "w:gz")
826	>	path=os.environ['CRABDIR'] + '/python/'
827	>	for file in ['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py', 'parseCrabFjr.py']:
828	>	tar.add(path+file,file)
829	>	common.logger.debug(5,"Files added to "+self.MLtgzfile+" : "+str(tar.getnames()))
830	>	tar.close()
831	>	except :
832	>	raise CrabException('Could not create ML files tar-ball')
833
834		return
835
836	+	def additionalInputFileTgz(self):
837	+	"""
838	+	Put all additional files into a tar ball and return its name
839	+	"""
840	+	import tarfile
841	+	tarName=  common.work_space.pathForTgz()+'share/'+self.additional_tgz_name
842	+	tar = tarfile.open(tarName, "w:gz")
843	+	for file in self.additional_inbox_files:
844	+	tar.add(file,string.split(file,'/')[-1])
845	+	common.logger.debug(5,"Files added to "+self.additional_tgz_name+" : "+str(tar.getnames()))
846	+	tar.close()
847	+	return tarName
848	+
849		def wsSetupEnvironment(self, nj):
850		"""
851		Returns part of a job script which prepares
#	Line 684 | Line 857 | class Cmssw(JobType):
857		## OLI_Daniele at this level  middleware already known
858
859		txt += 'if [ $middleware == LCG ]; then \n'
860	+	txt += '    echo "### First set SCRAM ARCH and BUILD_ARCH to ' + self.executable_arch + ' ###"\n'
861	+	txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
862	+	txt += '    export BUILD_ARCH='+self.executable_arch+'\n'
863		txt += self.wsSetupCMSLCGEnvironment_()
864		txt += 'elif [ $middleware == OSG ]; then\n'
865		txt += '    WORKING_DIR=`/bin/mktemp  -d $OSG_WN_TMP/cms_XXXXXXXXXXXX`\n'
866		txt += '    echo "Created working directory: $WORKING_DIR"\n'
867		txt += '    if [ ! -d $WORKING_DIR ] ;then\n'
868		txt += '        echo "SET_CMS_ENV 10016 ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
869	<	txt += '        echo "JOB_EXIT_STATUS = 10016"\n'
870	<	txt += '        echo "JobExitCode=10016" | tee -a $RUNTIME_AREA/$repo\n'
871	<	txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
869	>	txt += '    echo "JOB_EXIT_STATUS = 10016"\n'
870	>	txt += '    echo "JobExitCode=10016" | tee -a $RUNTIME_AREA/$repo\n'
871	>	txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
872		txt += '        rm -f $RUNTIME_AREA/$repo \n'
873		txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
874		txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
#	Line 702 | Line 878 | class Cmssw(JobType):
878		txt += '    echo "Change to working directory: $WORKING_DIR"\n'
879		txt += '    cd $WORKING_DIR\n'
880		txt += self.wsSetupCMSOSGEnvironment_()
881	+	txt += '    echo "### Set SCRAM ARCH to ' + self.executable_arch + ' ###"\n'
882	+	txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
883		txt += 'fi\n'
884
885		# Prepare JobType-specific part
#	Line 724 | Line 902 | class Cmssw(JobType):
902		txt += '        cd $RUNTIME_AREA\n'
903		txt += '        /bin/rm -rf $WORKING_DIR\n'
904		txt += '        if [ -d $WORKING_DIR ] ;then\n'
905	<	txt += '            echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
906	<	txt += '            echo "JOB_EXIT_STATUS = 10018"\n'
907	<	txt += '            echo "JobExitCode=10018" | tee -a $RUNTIME_AREA/$repo\n'
908	<	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
905	>	txt += '            echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
906	>	txt += '            echo "JOB_EXIT_STATUS = 10018"\n'
907	>	txt += '            echo "JobExitCode=10018" | tee -a $RUNTIME_AREA/$repo\n'
908	>	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
909		txt += '            rm -f $RUNTIME_AREA/$repo \n'
910		txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
911		txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
#	Line 737 | Line 915 | class Cmssw(JobType):
915		txt += 'fi \n'
916		txt += 'echo "CMSSW_VERSION =  '+self.version+'"\n'
917		txt += 'cd '+self.version+'\n'
918	+	########## FEDE FOR DBS2 ######################
919	+	txt += 'SOFTWARE_DIR=`pwd`\n'
920	+	txt += 'echo SOFTWARE_DIR=$SOFTWARE_DIR \n'
921	+	###############################################
922		### needed grep for bug in scramv1 ###
923	+	txt += scram+' runtime -sh\n'
924		txt += 'eval `'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME`\n'
925	+	txt += 'echo $PATH\n'
926
927		# Handle the arguments:
928		txt += "\n"
#	Line 760 | Line 944 | class Cmssw(JobType):
944		txt += '        cd $RUNTIME_AREA\n'
945		txt += '        /bin/rm -rf $WORKING_DIR\n'
946		txt += '        if [ -d $WORKING_DIR ] ;then\n'
947	<	txt += '            echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
948	<	txt += '            echo "JOB_EXIT_STATUS = 50114"\n'
949	<	txt += '            echo "JobExitCode=50114" | tee -a $RUNTIME_AREA/$repo\n'
950	<	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
947	>	txt += '            echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
948	>	txt += '            echo "JOB_EXIT_STATUS = 50114"\n'
949	>	txt += '            echo "JobExitCode=50114" | tee -a $RUNTIME_AREA/$repo\n'
950	>	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
951		txt += '            rm -f $RUNTIME_AREA/$repo \n'
952		txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
953		txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
#	Line 775 | Line 959 | class Cmssw(JobType):
959
960		# Prepare job-specific part
961		job = common.job_list[nj]
962	+	### FEDE FOR DBS OUTPUT PUBLICATION
963	+	if (self.datasetPath):
964	+	txt += '\n'
965	+	txt += 'DatasetPath='+self.datasetPath+'\n'
966	+
967	+	datasetpath_split = self.datasetPath.split("/")
968	+
969	+	txt += 'PrimaryDataset='+datasetpath_split[1]+'\n'
970	+	txt += 'DataTier='+datasetpath_split[2]+'\n'
971	+	#txt += 'ProcessedDataset='+datasetpath_split[3]+'\n'
972	+	txt += 'ApplicationFamily=cmsRun\n'
973	+
974	+	else:
975	+	txt += 'DatasetPath=MCDataTier\n'
976	+	txt += 'PrimaryDataset=null\n'
977	+	txt += 'DataTier=null\n'
978	+	#txt += 'ProcessedDataset=null\n'
979	+	txt += 'ApplicationFamily=MCDataTier\n'
980		if self.pset != None: #CarlosDaniele
981		pset = os.path.basename(job.configFilename())
982		txt += '\n'
983	+	txt += 'cp  $RUNTIME_AREA/'+pset+' .\n'
984		if (self.datasetPath): # standard job
985		#txt += 'InputFiles=$2\n'
986		txt += 'InputFiles=${args[1]}\n'
987		txt += 'MaxEvents=${args[2]}\n'
988		txt += 'SkipEvents=${args[3]}\n'
989		txt += 'echo "Inputfiles:<$InputFiles>"\n'
990	<	txt += 'sed "s#{\'INPUT\'}#$InputFiles#" $RUNTIME_AREA/'+pset+' > pset_tmp_1.cfg\n'
990	>	txt += 'sed "s#{\'INPUT\'}#$InputFiles#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
991		txt += 'echo "MaxEvents:<$MaxEvents>"\n'
992	<	txt += 'sed "s#INPUTMAXEVENTS#$MaxEvents#" pset_tmp_1.cfg > pset_tmp_2.cfg\n'
992	>	txt += 'sed "s#INPUTMAXEVENTS#$MaxEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
993		txt += 'echo "SkipEvents:<$SkipEvents>"\n'
994	<	txt += 'sed "s#INPUTSKIPEVENTS#$SkipEvents#" pset_tmp_2.cfg > pset.cfg\n'
994	>	txt += 'sed "s#INPUTSKIPEVENTS#$SkipEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
995		else:  # pythia like job
996	+	seedIndex=1
997	+	if (self.firstRun):
998	+	txt += 'FirstRun=${args['+str(seedIndex)+']}\n'
999	+	txt += 'echo "FirstRun: <$FirstRun>"\n'
1000	+	txt += 'sed "s#\<INPUTFIRSTRUN\>#$FirstRun#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1001	+	seedIndex=seedIndex+1
1002	+
1003		if (self.sourceSeed):
1004	<	#                    txt += 'Seed=$2\n'
1005	<	txt += 'Seed=${args[1]}\n'
1006	<	txt += 'echo "Seed: <$Seed>"\n'
1007	<	txt += 'sed "s#\<INPUT\>#$Seed#" $RUNTIME_AREA/'+pset+' > tmp.cfg\n'
1004	>	txt += 'Seed=${args['+str(seedIndex)+']}\n'
1005	>	txt += 'sed "s#\<INPUT\>#$Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1006	>	seedIndex=seedIndex+1
1007	>	## the following seeds are not always present
1008		if (self.sourceSeedVtx):
1009	<	#                        txt += 'VtxSeed=$3\n'
800	<	txt += 'VtxSeed=${args[2]}\n'
1009	>	txt += 'VtxSeed=${args['+str(seedIndex)+']}\n'
1010		txt += 'echo "VtxSeed: <$VtxSeed>"\n'
1011	<	txt += 'sed "s#INPUTVTX#$VtxSeed#" tmp.cfg > pset.cfg\n'
1012	<	else:
1013	<	txt += 'mv tmp.cfg pset.cfg\n'
1014	<	else:
1015	<	txt += '# Copy untouched pset\n'
1016	<	txt += 'cp $RUNTIME_AREA/'+pset+' pset.cfg\n'
1017	<
1011	>	txt += 'sed "s#\<INPUTVTX\>#$VtxSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1012	>	seedIndex += 1
1013	>	if (self.sourceSeedG4):
1014	>	txt += 'G4Seed=${args['+str(seedIndex)+']}\n'
1015	>	txt += 'echo "G4Seed: <$G4Seed>"\n'
1016	>	txt += 'sed "s#\<INPUTG4\>#$G4Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1017	>	seedIndex += 1
1018	>	if (self.sourceSeedMix):
1019	>	txt += 'mixSeed=${args['+str(seedIndex)+']}\n'
1020	>	txt += 'echo "MixSeed: <$mixSeed>"\n'
1021	>	txt += 'sed "s#\<INPUTMIX\>#$mixSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1022	>	seedIndex += 1
1023	>	pass
1024	>	pass
1025	>	txt += 'mv -f '+pset+' pset.cfg\n'
1026
1027		if len(self.additional_inbox_files) > 0:
1028	<	for file in self.additional_inbox_files:
1029	<	relFile = file.split("/")[-1]
1030	<	txt += 'if [ -e $RUNTIME_AREA/'+relFile+' ] ; then\n'
814	<	txt += '   cp $RUNTIME_AREA/'+relFile+' .\n'
815	<	txt += '   chmod +x '+relFile+'\n'
816	<	txt += 'fi\n'
1028	>	txt += 'if [ -e $RUNTIME_AREA/'+self.additional_tgz_name+' ] ; then\n'
1029	>	txt += '  tar xzvf $RUNTIME_AREA/'+self.additional_tgz_name+'\n'
1030	>	txt += 'fi\n'
1031		pass
1032
1033		if self.pset != None: #CarlosDaniele
#	Line 824 | Line 1038 | class Cmssw(JobType):
1038		txt += 'cat pset.cfg\n'
1039		txt += 'echo "****** end pset.cfg ********"\n'
1040		txt += '\n'
1041	+	### FEDE FOR DBS OUTPUT PUBLICATION
1042	+	txt += 'PSETHASH=`EdmConfigHash < pset.cfg` \n'
1043	+	txt += 'echo "PSETHASH = $PSETHASH" \n'
1044	+	##############
1045	+	txt += '\n'
1046		# txt += 'echo "***** cat pset1.cfg *********"\n'
1047		# txt += 'cat pset1.cfg\n'
1048		# txt += 'echo "****** end pset1.cfg ********"\n'
1049		return txt
1050
1051	<	def wsBuildExe(self, nj):
1051	>	def wsBuildExe(self, nj=0):
1052		"""
1053		Put in the script the commands to build an executable
1054		or a library.
#	Line 864 | Line 1083 | class Cmssw(JobType):
1083		txt += 'else \n'
1084		txt += '   echo "Successful untar" \n'
1085		txt += 'fi \n'
1086	+	txt += '\n'
1087	+	txt += 'echo "Include ProdAgentApi and PRODCOMMON in PYTHONPATH"\n'
1088	+	txt += 'if [ -z "$PYTHONPATH" ]; then\n'
1089	+	#### FEDE FOR DBS OUTPUT PUBLICATION
1090	+	txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdAgentApi:$SOFTWARE_DIR/ProdCommon\n'
1091	+	#txt += '   export PYTHONPATH=`pwd`/ProdAgentApi:`pwd`/ProdCommon\n'
1092	+	#txt += '   export PYTHONPATH=ProdAgentApi\n'
1093	+	txt += 'else\n'
1094	+	txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdAgentApi:$SOFTWARE_DIR/ProdCommon:${PYTHONPATH}\n'
1095	+	#txt += '   export PYTHONPATH=`pwd`/ProdAgentApi:`pwd`/ProdCommon:${PYTHONPATH}\n'
1096	+	#txt += '   export PYTHONPATH=ProdAgentApi:${PYTHONPATH}\n'
1097	+	txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
1098	+	###################
1099	+	txt += 'fi\n'
1100	+	txt += '\n'
1101	+
1102		pass
1103
1104		return txt
#	Line 875 | Line 1110 | class Cmssw(JobType):
1110		"""
1111
1112		def executableName(self):
1113	<	if self.pset == None: #CarlosDaniele
1113	>	if self.scriptExe: #CarlosDaniele
1114		return "sh "
1115		else:
1116		return self.executable
1117
1118		def executableArgs(self):
1119	<	if self.pset == None:#CarlosDaniele
1119	>	if self.scriptExe:#CarlosDaniele
1120		return   self.scriptExe + " $NJob"
1121	<	else:
1122	<	return " -p pset.cfg"
1121	>	else:
1122	>	# if >= CMSSW_1_5_X, add -e
1123	>	version_array = self.scram.getSWVersion().split('_')
1124	>	major = 0
1125	>	minor = 0
1126	>	try:
1127	>	major = int(version_array[1])
1128	>	minor = int(version_array[2])
1129	>	except:
1130	>	msg = "Cannot parse CMSSW version string: " + "_".join(version_array) + " for major and minor release number!"
1131	>	raise CrabException(msg)
1132	>	if major >= 1 and minor >= 5 :
1133	>	return " -e -p pset.cfg"
1134	>	else:
1135	>	return " -p pset.cfg"
1136
1137		def inputSandbox(self, nj):
1138		"""
1139		Returns a list of filenames to be put in JDL input sandbox.
1140		"""
1141		inp_box = []
1142	<	# dict added to delete duplicate from input sandbox file list
1143	<	seen = {}
1142	>	# # dict added to delete duplicate from input sandbox file list
1143	>	# seen = {}
1144		## code
1145		if os.path.isfile(self.tgzNameWithPath):
1146		inp_box.append(self.tgzNameWithPath)
1147	+	if os.path.isfile(self.MLtgzfile):
1148	+	inp_box.append(self.MLtgzfile)
1149		## config
1150	<	if not self.pset is None: #CarlosDaniele
1151	<	inp_box.append(common.job_list[nj].configFilename())
1150	>	if not self.pset is None:
1151	>	inp_box.append(common.work_space.pathForTgz() + 'job/' + self.configFilename())
1152		## additional input files
1153	<	#for file in self.additional_inbox_files:
1154	<	#    inp_box.append(common.work_space.cwdDir()+file)
1153	>	tgz = self.additionalInputFileTgz()
1154	>	inp_box.append(tgz)
1155		return inp_box
1156
1157		def outputSandbox(self, nj):
#	Line 910 | Line 1160 | class Cmssw(JobType):
1160		"""
1161		out_box = []
1162
913	–	stdout=common.job_list[nj].stdout()
914	–	stderr=common.job_list[nj].stderr()
915	–
1163		## User Declared output files
1164	<	for out in self.output_file:
1164	>	for out in (self.output_file+self.output_file_sandbox):
1165		n_out = nj + 1
1166		out_box.append(self.numberFile_(out,str(n_out)))
1167		return out_box
921	–	return []
1168
1169		def prepareSteeringCards(self):
1170		"""
#	Line 934 | Line 1180 | class Cmssw(JobType):
1180		txt = '\n'
1181		txt += '# directory content\n'
1182		txt += 'ls \n'
1183	<	file_list = ''
1184	<	for fileWithSuffix in self.output_file:
1183	>
1184	>	for fileWithSuffix in (self.output_file+self.output_file_sandbox):
1185		output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
940	–	file_list=file_list+output_file_num+' '
1186		txt += '\n'
1187		txt += '# check output file\n'
1188	<	txt += 'ls '+fileWithSuffix+'\n'
1189	<	txt += 'ls_result=$?\n'
1190	<	#txt += 'exe_result=$?\n'
1191	<	txt += 'if [ $ls_result -ne 0 ] ; then\n'
1192	<	txt += '   echo "ERROR: Problem with output file"\n'
1193	<	#txt += '   echo "JOB_EXIT_STATUS = $exe_result"\n'
1194	<	#txt += '   echo "JobExitCode=60302" | tee -a $RUNTIME_AREA/$repo\n'
1195	<	#txt += '   dumpStatus $RUNTIME_AREA/$repo\n'
1196	<	### OLI_DANIELE
1188	>	# txt += 'ls '+fileWithSuffix+'\n'
1189	>	# txt += 'ls_result=$?\n'
1190	>	txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1191	>	###### FEDE FOR OUTPUT DATA PUBLICATION ########
1192	>	txt += '    mv '+fileWithSuffix+' $RUNTIME_AREA\n'
1193	>	txt += '    cp $RUNTIME_AREA/'+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1194	>	################################################
1195	>	txt += 'else\n'
1196	>	txt += '    exit_status=60302\n'
1197	>	txt += '    echo "ERROR: Problem with output file '+fileWithSuffix+'"\n'
1198	>	############# FEDE ADDED CHECK FOR OUTPUT #############
1199	>	if fileWithSuffix in self.output_file:
1200	>	txt += '    echo "JOB_EXIT_STATUS = $exit_status"\n'
1201	>	txt += '    exit $exit_status\n'
1202	>	#######################################################
1203		if common.scheduler.boss_scheduler_name == 'condor_g':
1204		txt += '    if [ $middleware == OSG ]; then \n'
1205		txt += '        echo "prepare dummy output file"\n'
1206		txt += '        echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
1207		txt += '    fi \n'
957	–	txt += 'else\n'
958	–	txt += '   cp '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1208		txt += 'fi\n'
1209	<
1209	>	file_list = []
1210	>	for fileWithSuffix in (self.output_file):
1211	>	file_list.append(self.numberFile_(fileWithSuffix, '$NJob'))
1212	>
1213	>	txt += 'file_list="'+string.join(file_list,' ')+'"\n'
1214		txt += 'cd $RUNTIME_AREA\n'
962	–	file_list=file_list[:-1]
963	–	txt += 'file_list="'+file_list+'"\n'
964	–	txt += 'cd $RUNTIME_AREA\n'
965	–	### OLI_DANIELE
966	–	txt += 'if [ $middleware == OSG ]; then\n'
967	–	txt += '    cd $RUNTIME_AREA\n'
968	–	txt += '    echo "Remove working directory: $WORKING_DIR"\n'
969	–	txt += '    /bin/rm -rf $WORKING_DIR\n'
970	–	txt += '    if [ -d $WORKING_DIR ] ;then\n'
971	–	txt += '        echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
972	–	txt += '        echo "JOB_EXIT_STATUS = 60999"\n'
973	–	txt += '        echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
974	–	txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
975	–	txt += '        rm -f $RUNTIME_AREA/$repo \n'
976	–	txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
977	–	txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
978	–	txt += '    fi\n'
979	–	txt += 'fi\n'
980	–	txt += '\n'
1215		return txt
1216
1217		def numberFile_(self, file, txt):
#	Line 988 | Line 1222 | class Cmssw(JobType):
1222		# take away last extension
1223		name = p[0]
1224		for x in p[1:-1]:
1225	<	name=name+"."+x
1225	>	name=name+"."+x
1226		# add "_txt"
1227		if len(p)>1:
1228	<	ext = p[len(p)-1]
1229	<	result = name + '_' + txt + "." + ext
1228	>	ext = p[len(p)-1]
1229	>	result = name + '_' + txt + "." + ext
1230		else:
1231	<	result = name + '_' + txt
1231	>	result = name + '_' + txt
1232
1233		return result
1234
1235	<	def getRequirements(self):
1235	>	def getRequirements(self, nj=[]):
1236		"""
1237		return job requirements to add to jdl files
1238		"""
#	Line 1007 | Line 1241 | class Cmssw(JobType):
1241		req='Member("VO-cms-' + \
1242		self.version + \
1243		'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1244	+	## SL add requirement for OS version only if SL4
1245	+	#reSL4 = re.compile( r'slc4' )
1246	+	if self.executable_arch: # and reSL4.search(self.executable_arch):
1247	+	req+=' && Member("VO-cms-' + \
1248	+	self.executable_arch + \
1249	+	'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1250
1251		req = req + ' && (other.GlueHostNetworkAdapterOutboundIP)'
1252
#	Line 1026 | Line 1266 | class Cmssw(JobType):
1266		txt += '   echo "### SETUP CMS OSG  ENVIRONMENT ###"\n'
1267		txt += '   if [ -f $GRID3_APP_DIR/cmssoft/cmsset_default.sh ] ;then\n'
1268		txt += '      # Use $GRID3_APP_DIR/cmssoft/cmsset_default.sh to setup cms software\n'
1269	+	txt += '       export SCRAM_ARCH='+self.executable_arch+'\n'
1270		txt += '       source $GRID3_APP_DIR/cmssoft/cmsset_default.sh '+self.version+'\n'
1271		txt += '   elif [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n'
1272		txt += '      # Use $OSG_APP/cmssoft/cms/cmsset_default.sh to setup cms software\n'
1273	+	txt += '       export SCRAM_ARCH='+self.executable_arch+'\n'
1274		txt += '       source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1275		txt += '   else\n'
1276		txt += '       echo "SET_CMS_ENV 10020 ==> ERROR $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
#	Line 1044 | Line 1286 | class Cmssw(JobType):
1286		txt += '       cd $RUNTIME_AREA\n'
1287		txt += '       /bin/rm -rf $WORKING_DIR\n'
1288		txt += '       if [ -d $WORKING_DIR ] ;then\n'
1289	<	txt += '            echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1290	<	txt += '            echo "JOB_EXIT_STATUS = 10017"\n'
1291	<	txt += '            echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n'
1292	<	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
1293	<	txt += '            rm -f $RUNTIME_AREA/$repo \n'
1294	<	txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1295	<	txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1289	>	txt += '           echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1290	>	txt += '           echo "JOB_EXIT_STATUS = 10017"\n'
1291	>	txt += '           echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n'
1292	>	txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1293	>	txt += '           rm -f $RUNTIME_AREA/$repo \n'
1294	>	txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1295	>	txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1296		txt += '       fi\n'
1297		txt += '\n'
1298		txt += '       exit 1\n'
#	Line 1105 | Line 1347 | class Cmssw(JobType):
1347		txt += '       fi\n'
1348		txt += '   fi\n'
1349		txt += '   \n'
1108	–	txt += '   string=`cat /etc/redhat-release`\n'
1109	–	txt += '   echo $string\n'
1110	–	txt += '   if [[ $string = *alhalla* ]]; then\n'
1111	–	txt += '       echo "SCRAM_ARCH= $SCRAM_ARCH"\n'
1112	–	txt += '   elif [[ $string = *Enterprise* ]] || [[ $string = *cientific* ]]; then\n'
1113	–	txt += '       export SCRAM_ARCH=slc3_ia32_gcc323\n'
1114	–	txt += '       echo "SCRAM_ARCH= $SCRAM_ARCH"\n'
1115	–	txt += '   else\n'
1116	–	txt += '       echo "SET_CMS_ENV 10033 ==> ERROR OS unknown, LCG environment not initialized"\n'
1117	–	txt += '       echo "JOB_EXIT_STATUS = 10033"\n'
1118	–	txt += '       echo "JobExitCode=10033" | tee -a $RUNTIME_AREA/$repo\n'
1119	–	txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
1120	–	txt += '       rm -f $RUNTIME_AREA/$repo \n'
1121	–	txt += '       echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1122	–	txt += '       echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1123	–	txt += '       exit 1\n'
1124	–	txt += '   fi\n'
1350		txt += '   echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1351		txt += '   echo "### END SETUP CMS LCG ENVIRONMENT ###"\n'
1352		return txt
1353
1354	+	### FEDE FOR DBS OUTPUT PUBLICATION
1355	+	def modifyReport(self, nj):
1356	+	"""
1357	+	insert the part of the script that modifies the FrameworkJob Report
1358	+	"""
1359	+
1360	+	txt = ''
1361	+	txt += 'echo "Modify Job Report" \n'
1362	+	#txt += 'chmod a+x $RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1363	+	################ FEDE FOR DBS2 #############################################
1364	+	txt += 'chmod a+x $SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1365	+	#############################################################################
1366	+	try:
1367	+	publish_data = int(self.cfg_params['USER.publish_data'])
1368	+	except KeyError:
1369	+	publish_data = 0
1370	+
1371	+	txt += 'if [ -z "$SE" ]; then\n'
1372	+	txt += '    SE="" \n'
1373	+	txt += 'fi \n'
1374	+	txt += 'if [ -z "$SE_PATH" ]; then\n'
1375	+	txt += '    SE_PATH="" \n'
1376	+	txt += 'fi \n'
1377	+	txt += 'echo "SE = $SE"\n'
1378	+	txt += 'echo "SE_PATH = $SE_PATH"\n'
1379	+
1380	+	if (publish_data == 1):
1381	+	#processedDataset = self.cfg_params['USER.processed_datasetname']
1382	+	processedDataset = self.cfg_params['USER.publish_data_name']
1383	+	txt += 'ProcessedDataset='+processedDataset+'\n'
1384	+	#### LFN=/store/user/<user>/processedDataset_PSETHASH
1385	+	txt += 'if [ "$SE_PATH" == "" ]; then\n'
1386	+	#### FEDE: added slash in LFN ##############
1387	+	txt += '    FOR_LFN=/copy_problems/ \n'
1388	+	txt += 'else \n'
1389	+	txt += '    tmp=`echo $SE_PATH | awk -F \'store\' \'{print$2}\'` \n'
1390	+	#####  FEDE TO BE CHANGED, BECAUSE STORE IS HARDCODED!!!! ########
1391	+	txt += '    FOR_LFN=/store$tmp \n'
1392	+	txt += 'fi \n'
1393	+	txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1394	+	txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1395	+	txt += 'echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n'
1396	+	#txt += 'echo "$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1397	+	txt += 'echo "$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1398	+	txt += '$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1399	+	#txt += '$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1400	+
1401	+	txt += 'modifyReport_result=$?\n'
1402	+	txt += 'echo modifyReport_result = $modifyReport_result\n'
1403	+	txt += 'if [ $modifyReport_result -ne 0 ]; then\n'
1404	+	txt += '    exit_status=1\n'
1405	+	txt += '    echo "ERROR: Problem with ModifyJobReport"\n'
1406	+	txt += 'else\n'
1407	+	txt += '    mv NewFrameworkJobReport.xml crab_fjr_$NJob.xml\n'
1408	+	txt += 'fi\n'
1409	+	else:
1410	+	txt += 'ProcessedDataset=no_data_to_publish \n'
1411	+	#### FEDE: added slash in LFN ##############
1412	+	txt += 'FOR_LFN=/local/ \n'
1413	+	txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1414	+	txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1415	+	return txt
1416	+
1417	+	def cleanEnv(self):
1418	+	### OLI_DANIELE
1419	+	txt = ''
1420	+	txt += 'if [ $middleware == OSG ]; then\n'
1421	+	txt += '    cd $RUNTIME_AREA\n'
1422	+	txt += '    echo "Remove working directory: $WORKING_DIR"\n'
1423	+	txt += '    /bin/rm -rf $WORKING_DIR\n'
1424	+	txt += '    if [ -d $WORKING_DIR ] ;then\n'
1425	+	txt += '              echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1426	+	txt += '              echo "JOB_EXIT_STATUS = 60999"\n'
1427	+	txt += '              echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
1428	+	txt += '              dumpStatus $RUNTIME_AREA/$repo\n'
1429	+	txt += '        rm -f $RUNTIME_AREA/$repo \n'
1430	+	txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1431	+	txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1432	+	txt += '    fi\n'
1433	+	txt += 'fi\n'
1434	+	txt += '\n'
1435	+	return txt
1436	+
1437		def setParam_(self, param, value):
1438		self._params[param] = value
1439
#	Line 1138 | Line 1446 | class Cmssw(JobType):
1446		def getTaskid(self):
1447		return self._taskId
1448
1141	–	#######################################################################
1449		def uniquelist(self, old):
1450		"""
1451		remove duplicates from a list

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines