ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/cms_cmssw.py

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.14 by gutsche, Tue Jun 27 15:39:58 2006 UTC vs.
Revision 1.55 by slacapra, Wed Nov 22 14:57:26 2006 UTC

#	Line 2 | Line 2 | from JobType import JobType
2		from crab_logger import Logger
3		from crab_exceptions import *
4		from crab_util import *
5	+	import math
6		import common
7		import PsetManipulator
8
9	<	import DBSInfo_EDM
10	<	import DataDiscovery_EDM
11	<	import DataLocation_EDM
9	>	import DBSInfo
10	>	import DataDiscovery
11	>	import DataLocation
12		import Scram
13
14	<	import os, string, re
14	>	import glob, os, string, re
15
16		class Cmssw(JobType):
17	<	def __init__(self, cfg_params):
17	>	def __init__(self, cfg_params, ncjobs):
18		JobType.__init__(self, 'CMSSW')
19		common.logger.debug(3,'CMSSW::__init__')
20
20	–	self.analisys_common_info = {}
21		# Marco.
22		self._params = {}
23		self.cfg_params = cfg_params
24	+
25	+	# number of jobs requested to be created, limit obj splitting
26	+	self.ncjobs = ncjobs
27	+
28		log = common.logger
29
30		self.scram = Scram.Scram(cfg_params)
#	Line 29 | Line 33 | class Cmssw(JobType):
33		self.scriptExe = ''
34		self.executable = ''
35		self.tgz_name = 'default.tgz'
36	+	self.pset = ''      #scrip use case Da
37	+	self.datasetPath = '' #scrip use case Da
38
39	+	# set FJR file name
40	+	self.fjrFileName = 'crab_fjr.xml'
41
42		self.version = self.scram.getSWVersion()
43	+	common.taskDB.setDict('codeVersion',self.version)
44		self.setParam_('application', self.version)
36	–	common.analisys_common_info['sw_version'] = self.version
37	–	### FEDE
38	–	common.analisys_common_info['copy_input_data'] = 0
39	–	common.analisys_common_info['events_management'] = 1
45
46		### collect Data cards
47		try:
#	Line 44 | Line 49 | class Cmssw(JobType):
49		log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
50		if string.lower(tmp)=='none':
51		self.datasetPath = None
52	+	self.selectNoInput = 1
53		else:
54		self.datasetPath = tmp
55	+	self.selectNoInput = 0
56		except KeyError:
57		msg = "Error: datasetpath not defined "
58		raise CrabException(msg)
#	Line 82 | Line 89 | class Cmssw(JobType):
89		try:
90		self.pset = cfg_params['CMSSW.pset']
91		log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset)
92	<	if (not os.path.exists(self.pset)):
93	<	raise CrabException("User defined PSet file "+self.pset+" does not exist")
92	>	if self.pset.lower() != 'none' :
93	>	if (not os.path.exists(self.pset)):
94	>	raise CrabException("User defined PSet file "+self.pset+" does not exist")
95	>	else:
96	>	self.pset = None
97		except KeyError:
98		raise CrabException("PSet file missing. Cannot run cmsRun ")
99
100		# output files
101	+	## stuff which must be returned always via sandbox
102	+	self.output_file_sandbox = []
103	+
104	+	# add fjr report by default via sandbox
105	+	self.output_file_sandbox.append(self.fjrFileName)
106	+
107	+	# other output files to be returned via sandbox or copied to SE
108		try:
109		self.output_file = []
110
111	+
112		tmp = cfg_params['CMSSW.output_file']
113		if tmp != '':
114		tmpOutFiles = string.split(cfg_params['CMSSW.output_file'],',')
#	Line 100 | Line 118 | class Cmssw(JobType):
118		self.output_file.append(tmp)
119		pass
120		else:
121	<	log.message("No output file defined: only stdout/err will be available")
121	>	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available")
122		pass
123		pass
124		except KeyError:
125	<	log.message("No output file defined: only stdout/err will be available")
125	>	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available")
126		pass
127
128		# script_exe file as additional file in inputSandbox
129		try:
130		self.scriptExe = cfg_params['USER.script_exe']
113	–	self.additional_inbox_files.append(self.scriptExe)
131		if self.scriptExe != '':
132		if not os.path.isfile(self.scriptExe):
133		msg ="WARNING. file "+self.scriptExe+" not found"
134		raise CrabException(msg)
135	+	self.additional_inbox_files.append(string.strip(self.scriptExe))
136		except KeyError:
137	<	pass
138	<
137	>	self.scriptExe = ''
138	>	#CarlosDaniele
139	>	if self.datasetPath == None and self.pset == None and self.scriptExe == '' :
140	>	msg ="WARNING. script_exe  not defined"
141	>	raise CrabException(msg)
142	>
143		## additional input files
144		try:
145	<	tmpAddFiles = string.split(cfg_params['CMSSW.additional_input_files'],',')
145	>	tmpAddFiles = string.split(cfg_params['USER.additional_input_files'],',')
146		for tmp in tmpAddFiles:
147	<	if not os.path.exists(tmp):
148	<	raise CrabException("Additional input file not found: "+tmp)
149	<	tmp=string.strip(tmp)
150	<	self.additional_inbox_files.append(tmp)
147	>	tmp = string.strip(tmp)
148	>	dirname = ''
149	>	if not tmp[0]=="/": dirname = "."
150	>	files = glob.glob(os.path.join(dirname, tmp))
151	>	for file in files:
152	>	if not os.path.exists(file):
153	>	raise CrabException("Additional input file not found: "+file)
154	>	pass
155	>	self.additional_inbox_files.append(string.strip(file))
156		pass
157		pass
158	+	common.logger.debug(5,"Additional input files: "+str(self.additional_inbox_files))
159		except KeyError:
160		pass
161
162		# files per job
163		try:
164	<	self.filesPerJob = int(cfg_params['CMSSW.files_per_jobs']) #Daniele
165	<	self.selectFilesPerJob = 1
164	>	if (cfg_params['CMSSW.files_per_jobs']):
165	>	raise CrabException("files_per_jobs no longer supported.  Quitting.")
166		except KeyError:
167	<	self.filesPerJob = 0
140	<	self.selectFilesPerJob = 0
167	>	pass
168
169		## Events per job
170		try:
#	Line 147 | Line 174 | class Cmssw(JobType):
174		self.eventsPerJob = -1
175		self.selectEventsPerJob = 0
176
177	<	# To be implemented
178	<	# ## number of jobs
179	<	# try:
180	<	#     self.numberOfJobs =int( cfg_params['CMSSW.number_of_job'])
181	<	#     self.selectNumberOfJobs = 1
182	<	# except KeyError:
183	<	#     self.selectNumberOfJobs = 0
157	<
158	<	if (self.selectFilesPerJob == self.selectEventsPerJob):
159	<	msg = 'Must define either files_per_jobs or events_per_job'
160	<	raise CrabException(msg)
177	>	## number of jobs
178	>	try:
179	>	self.theNumberOfJobs =int( cfg_params['CMSSW.number_of_jobs'])
180	>	self.selectNumberOfJobs = 1
181	>	except KeyError:
182	>	self.theNumberOfJobs = 0
183	>	self.selectNumberOfJobs = 0
184
162	–	if (self.selectEventsPerJob  and not self.datasetPath == None):
163	–	msg = 'Splitting according to events_per_job available only with None as datasetpath'
164	–	raise CrabException(msg)
165	–
185		try:
186		self.total_number_of_events = int(cfg_params['CMSSW.total_number_of_events'])
187	+	self.selectTotalNumberEvents = 1
188		except KeyError:
189	<	msg = 'Must define total_number_of_events'
190	<	raise CrabException(msg)
191	<
192	<	CEBlackList = []
189	>	self.total_number_of_events = 0
190	>	self.selectTotalNumberEvents = 0
191	>
192	>	if self.pset != None: #CarlosDaniele
193	>	if ( (self.selectTotalNumberEvents + self.selectEventsPerJob + self.selectNumberOfJobs) != 2 ):
194	>	msg = 'Must define exactly two of total_number_of_events, events_per_job, or number_of_jobs.'
195	>	raise CrabException(msg)
196	>	else:
197	>	if (self.selectNumberOfJobs == 0):
198	>	msg = 'Must specify  number_of_jobs.'
199	>	raise CrabException(msg)
200	>
201	>	## source seed for pythia
202		try:
203	<	tmpBad = string.split(cfg_params['EDG.ce_black_list'],',')
175	<	for tmp in tmpBad:
176	<	tmp=string.strip(tmp)
177	<	CEBlackList.append(tmp)
203	>	self.sourceSeed = int(cfg_params['CMSSW.pythia_seed'])
204		except KeyError:
205	<	pass
205	>	self.sourceSeed = None
206	>	common.logger.debug(5,"No seed given")
207
208	<	self.reCEBlackList=[]
209	<	for bad in CEBlackList:
183	<	self.reCEBlackList.append(re.compile( bad ))
184	<
185	<	common.logger.debug(5,'CEBlackList: '+str(CEBlackList))
186	<
187	<	CEWhiteList = []
188	<	try:
189	<	tmpGood = string.split(cfg_params['EDG.ce_white_list'],',')
190	<	for tmp in tmpGood:
191	<	tmp=string.strip(tmp)
192	<	CEWhiteList.append(tmp)
208	>	try:
209	>	self.sourceSeedVtx = int(cfg_params['CMSSW.vtx_seed'])
210		except KeyError:
211	<	pass
212	<
213	<	#print 'CEWhiteList: ',CEWhiteList
214	<	self.reCEWhiteList=[]
198	<	for Good in CEWhiteList:
199	<	self.reCEWhiteList.append(re.compile( Good ))
200	<
201	<	common.logger.debug(5,'CEWhiteList: '+str(CEWhiteList))
202	<
203	<	self.PsetEdit = PsetManipulator.PsetManipulator(self.pset) #Daniele Pset
211	>	self.sourceSeedVtx = None
212	>	common.logger.debug(5,"No vertex seed given")
213	>	if self.pset != None: #CarlosDaniele
214	>	self.PsetEdit = PsetManipulator.PsetManipulator(self.pset) #Daniele Pset
215
216		#DBSDLS-start
217		## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
218		self.maxEvents=0  # max events available   ( --> check the requested nb. of evts in Creator.py)
219		self.DBSPaths={}  # all dbs paths requested ( --> input to the site local discovery script)
220	+	self.jobDestination=[]  # Site destination(s) for each job (list of lists)
221		## Perform the data location and discovery (based on DBS/DLS)
222		## SL: Don't if NONE is specified as input (pythia use case)
223	<	common.analisys_common_info['sites']=None
223	>	blockSites = {}
224		if self.datasetPath:
225	<	self.DataDiscoveryAndLocation(cfg_params)
225	>	blockSites = self.DataDiscoveryAndLocation(cfg_params)
226		#DBSDLS-end
227
228		self.tgzNameWithPath = self.getTarBall(self.executable)
217	–
218	–	# modify Pset
219	–	if (self.datasetPath): # standard job
220	–	self.PsetEdit.maxEvent(self.eventsPerJob) #Daniele
221	–	self.PsetEdit.inputModule("INPUT") #Daniele
222	–
223	–	else:  # pythia like job
224	–	self.PsetEdit.maxEvent(self.eventsPerJob) #Daniele
225	–	self.PsetEdit.pythiaSeed("INPUT") #Daniele
226	–	try:
227	–	self.sourceSeed = int(cfg_params['CMSSW.pythia_seed'])
228	–	except KeyError:
229	–	self.sourceSeed = 123456
230	–	common.logger.message("No seed given, will use "+str(self.sourceSeed))
231	–
232	–	self.PsetEdit.psetWriter(self.configFilename())
229
230		## Select Splitting
231	<	if self.selectFilesPerJob: self.jobSplittingPerFiles()
232	<	elif self.selectEventsPerJob: self.jobSplittingPerEvents()
233	<	else:
234	<	msg = 'Don\'t know how to split...'
235	<	raise CrabException(msg)
231	>	if self.selectNoInput:
232	>	if self.pset == None: #CarlosDaniele
233	>	self.jobSplittingForScript()
234	>	else:
235	>	self.jobSplittingNoInput()
236	>	else: self.jobSplittingByBlocks(blockSites)
237
238	+	# modify Pset
239	+	if self.pset != None: #CarlosDaniele
240	+	try:
241	+	if (self.datasetPath): # standard job
242	+	# allow to processa a fraction of events in a file
243	+	self.PsetEdit.inputModule("INPUT")
244	+	self.PsetEdit.maxEvent("INPUTMAXEVENTS")
245	+	self.PsetEdit.skipEvent("INPUTSKIPEVENTS")
246	+	else:  # pythia like job
247	+	self.PsetEdit.maxEvent(self.eventsPerJob)
248	+	if (self.sourceSeed) :
249	+	self.PsetEdit.pythiaSeed("INPUT")
250	+	if (self.sourceSeedVtx) :
251	+	self.PsetEdit.pythiaSeedVtx("INPUTVTX")
252	+	# add FrameworkJobReport to parameter-set
253	+	self.PsetEdit.addCrabFJR(self.fjrFileName)
254	+	self.PsetEdit.psetWriter(self.configFilename())
255	+	except:
256	+	msg='Error while manipuliating ParameterSet: exiting...'
257	+	raise CrabException(msg)
258
259		def DataDiscoveryAndLocation(self, cfg_params):
260
#	Line 250 | Line 267 | class Cmssw(JobType):
267		dataTiers = dataTiersList.split(',')
268
269		## Contact the DBS
270	+	common.logger.message("Contacting DBS...")
271		try:
272	<	self.pubdata=DataDiscovery_EDM.DataDiscovery_EDM(datasetPath, dataTiers, cfg_params)
272	>	self.pubdata=DataDiscovery.DataDiscovery(datasetPath, dataTiers, cfg_params)
273		self.pubdata.fetchDBSInfo()
274
275	<	except DataDiscovery_EDM.NotExistingDatasetError, ex :
275	>	except DataDiscovery.NotExistingDatasetError, ex :
276		msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
277		raise CrabException(msg)
278
279	<	except DataDiscovery_EDM.NoDataTierinProvenanceError, ex :
279	>	except DataDiscovery.NoDataTierinProvenanceError, ex :
280		msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
281		raise CrabException(msg)
282	<	except DataDiscovery_EDM.DataDiscoveryError, ex:
282	>	except DataDiscovery.DataDiscoveryError, ex:
283		msg = 'ERROR ***: failed Data Discovery in DBS  %s'%ex.getErrorMessage()
284		raise CrabException(msg)
285
#	Line 269 | Line 287 | class Cmssw(JobType):
287		## self.DBSPaths=self.pubdata.getDBSPaths()
288		common.logger.message("Required data are :"+self.datasetPath)
289
290	<	filesbyblock=self.pubdata.getFiles()
291	<	self.AllInputFiles=filesbyblock.values()
292	<	self.files = self.AllInputFiles
293	<
294	<	## TEMP
295	<	#    self.filesTmp = filesbyblock.values()
278	<	#    self.files = []
279	<	#    locPath='rfio:cmsbose2.bo.infn.it:/flatfiles/SE00/cms/fanfani/ProdTest/'
280	<	#    locPath=''
281	<	#    tmp = []
282	<	#    for file in self.filesTmp[0]:
283	<	#        tmp.append(locPath+file)
284	<	#    self.files.append(tmp)
285	<	## END TEMP
290	>	self.filesbyblock=self.pubdata.getFiles()
291	>	self.eventsbyblock=self.pubdata.getEventsPerBlock()
292	>	self.eventsbyfile=self.pubdata.getEventsPerFile()
293	>	# print str(self.filesbyblock)
294	>	# print 'self.eventsbyfile',len(self.eventsbyfile)
295	>	# print str(self.eventsbyfile)
296
297		## get max number of events
288	–	#common.logger.debug(10,"number of events for primary fileblocks %i"%self.pubdata.getMaxEvents())
298		self.maxEvents=self.pubdata.getMaxEvents() ##  self.maxEvents used in Creator.py
299	<	common.logger.message("\nThe number of available events is %s"%self.maxEvents)
299	>	common.logger.message("The number of available events is %s\n"%self.maxEvents)
300
301	+	common.logger.message("Contacting DLS...")
302		## Contact the DLS and build a list of sites hosting the fileblocks
303		try:
304	<	dataloc=DataLocation_EDM.DataLocation_EDM(filesbyblock.keys(),cfg_params)
304	>	dataloc=DataLocation.DataLocation(self.filesbyblock.keys(),cfg_params)
305		dataloc.fetchDLSInfo()
306	<	except DataLocation_EDM.DataLocationError , ex:
306	>	except DataLocation.DataLocationError , ex:
307		msg = 'ERROR ***: failed Data Location in DLS \n %s '%ex.getErrorMessage()
308		raise CrabException(msg)
309
300	–	allsites=dataloc.getSites()
301	–	common.logger.debug(5,"sites are %s"%allsites)
302	–	sites=self.checkBlackList(allsites)
303	–	common.logger.debug(5,"sites are (after black list) %s"%sites)
304	–	sites=self.checkWhiteList(sites)
305	–	common.logger.debug(5,"sites are (after white list) %s"%sites)
310
311	<	if len(sites)==0:
312	<	msg = 'No sites hosting all the needed data! Exiting... '
313	<	raise CrabException(msg)
311	>	sites = dataloc.getSites()
312	>	allSites = []
313	>	listSites = sites.values()
314	>	for list in listSites:
315	>	for oneSite in list:
316	>	allSites.append(oneSite)
317	>	allSites = self.uniquelist(allSites)
318
319	<	common.logger.message("List of Sites hosting the data : "+str(sites))
320	<	common.logger.debug(6, "List of Sites: "+str(sites))
321	<	common.analisys_common_info['sites']=sites    ## used in SchedulerEdg.py in createSchScript
314	<	self.setParam_('TargetCE', ','.join(sites))
315	<	return
319	>	common.logger.message("Sites ("+str(len(allSites))+") hosting part/all of dataset: "+str(allSites))
320	>	common.logger.debug(6, "List of Sites: "+str(allSites))
321	>	return sites
322
323	<	def jobSplittingPerFiles(self):
323	>	def jobSplittingByBlocks(self, blockSites):
324		"""
325	<	Perform job splitting based on number of files to be accessed per job
326	<	"""
327	<	common.logger.debug(5,'Splitting per input files')
328	<	common.logger.message('Required '+str(self.filesPerJob)+' files per job ')
329	<	common.logger.message('Required '+str(self.total_number_of_events)+' events in total ')
330	<
331	<	## TODO: SL need to have (from DBS) a detailed list of how many events per each file
332	<	n_tot_files = (len(self.files[0]))
333	<	#print "n_tot_files = ", n_tot_files
334	<	## SL: this is wrong if the files have different number of events
335	<	#print "self.maxEvents = ", self.maxEvents
336	<	evPerFile = int(self.maxEvents)/n_tot_files
337	<	#print "evPerFile = int(self.maxEvents)/n_tot_files =  ", evPerFile
338	<
339	<	common.logger.debug(5,'Events per File '+str(evPerFile))
340	<
341	<	## if asked to process all events, do it
342	<	if self.total_number_of_events == -1:
343	<	self.total_number_of_events=self.maxEvents
344	<	self.total_number_of_jobs = int(n_tot_files)*1/int(self.filesPerJob)
345	<	common.logger.message(str(self.total_number_of_jobs)+' jobs will be created for all available events '+str(self.total_number_of_events)+' events')
346	<
325	>	Perform job splitting. Jobs run over an integer number of files
326	>	and no more than one block.
327	>	ARGUMENT: blockSites: dictionary with blocks as keys and list of host sites as values
328	>	REQUIRES: self.selectTotalNumberEvents, self.selectEventsPerJob, self.selectNumberofJobs,
329	>	self.total_number_of_events, self.eventsPerJob, self.theNumberOfJobs,
330	>	self.maxEvents, self.filesbyblock
331	>	SETS: self.jobDestination - Site destination(s) for each job (a list of lists)
332	>	self.total_number_of_jobs - Total # of jobs
333	>	self.list_of_args - File(s) job will run on (a list of lists)
334	>	"""
335	>
336	>	# ---- Handle the possible job splitting configurations ---- #
337	>	if (self.selectTotalNumberEvents):
338	>	totalEventsRequested = self.total_number_of_events
339	>	if (self.selectEventsPerJob):
340	>	eventsPerJobRequested = self.eventsPerJob
341	>	if (self.selectNumberOfJobs):
342	>	totalEventsRequested = self.theNumberOfJobs * self.eventsPerJob
343	>
344	>	# If user requested all the events in the dataset
345	>	if (totalEventsRequested == -1):
346	>	eventsRemaining=self.maxEvents
347	>	# If user requested more events than are in the dataset
348	>	elif (totalEventsRequested > self.maxEvents):
349	>	eventsRemaining = self.maxEvents
350	>	common.logger.message("Requested "+str(self.total_number_of_events)+ " events, but only "+str(self.maxEvents)+" events are available.")
351	>	# If user requested less events than are in the dataset
352		else:
353	<	#print "self.total_number_of_events = ", self.total_number_of_events
354	<	#print "evPerFile = ", evPerFile
355	<	self.total_number_of_files = int(self.total_number_of_events/evPerFile)
356	<	#print "self.total_number_of_files = int(self.total_number_of_events/evPerFile) = " , self.total_number_of_files
357	<	## SL: if ask for less event than what is computed to be available on a
358	<	##     file, process the first file anyhow.
359	<	if self.total_number_of_files == 0:
360	<	self.total_number_of_files = self.total_number_of_files + 1
361	<
353	>	eventsRemaining = totalEventsRequested
354	>
355	>	# If user requested more events per job than are in the dataset
356	>	if (self.selectEventsPerJob and eventsPerJobRequested > self.maxEvents):
357	>	eventsPerJobRequested = self.maxEvents
358	>
359	>	# For user info at end
360	>	totalEventCount = 0
361	>
362	>	if (self.selectTotalNumberEvents and self.selectNumberOfJobs):
363	>	eventsPerJobRequested = int(eventsRemaining/self.theNumberOfJobs)
364
365	<	common.logger.debug(5,'N files  '+str(self.total_number_of_files))
365	>	if (self.selectNumberOfJobs):
366	>	common.logger.message("May not create the exact number_of_jobs requested.")
367
368	<	check = 0
368	>	if ( self.ncjobs == 'all' ) :
369	>	totalNumberOfJobs = 999999999
370	>	else :
371	>	totalNumberOfJobs = self.ncjobs
372
356	–	## Compute the number of jobs
357	–	#self.total_number_of_jobs = int(n_tot_files)*1/int(self.filesPerJob)
358	–	#print "self.total_number_of_files = ", self.total_number_of_files
359	–	#print "self.filesPerJob = ", self.filesPerJob
360	–	self.total_number_of_jobs = int(self.total_number_of_files/self.filesPerJob)
361	–	#print "self.total_number_of_jobs = ", self.total_number_of_jobs
362	–	common.logger.debug(5,'N jobs  '+str(self.total_number_of_jobs))
363	–
364	–	## is there any remainder?
365	–	check = int(self.total_number_of_files) - (int(self.total_number_of_jobs)*self.filesPerJob)
366	–
367	–	common.logger.debug(5,'Check  '+str(check))
368	–
369	–	if check > 0:
370	–	self.total_number_of_jobs =  self.total_number_of_jobs + 1
371	–	common.logger.message('Warning: last job will be created with '+str(check)+' files')
373
374	<	#common.logger.message(str(self.total_number_of_jobs)+' jobs will be created for a total of '+str((self.total_number_of_jobs-1)*self.filesPerJob*evPerFile + check*evPerFile)+' events')
375	<	common.logger.message(str(self.total_number_of_jobs)+' jobs will be created for a total of '+str((self.total_number_of_jobs)*self.filesPerJob*evPerFile + check*evPerFile)+' events')
376	<	pass
374	>	blocks = blockSites.keys()
375	>	blockCount = 0
376	>	# Backup variable in case self.maxEvents counted events in a non-included block
377	>	numBlocksInDataset = len(blocks)
378
379	+	jobCount = 0
380		list_of_lists = []
381	<	for i in xrange(0, int(n_tot_files), self.filesPerJob):
382	<	parString = "\\{"
383	<
384	<	params = self.files[0][i: i+self.filesPerJob]
385	<	for i in range(len(params) - 1):
386	<	parString += '\\\"' + params[i] + '\\\"\,'
381	>
382	>	# ---- Iterate over the blocks in the dataset until ---- #
383	>	# ---- we've met the requested total # of events    ---- #
384	>	while ( (eventsRemaining > 0) and (blockCount < numBlocksInDataset) and (jobCount < totalNumberOfJobs)):
385	>	block = blocks[blockCount]
386	>	blockCount += 1
387
385	–	parString += '\\\"' + params[len(params) - 1] + '\\\"\\}'
386	–	list_of_lists.append(parString)
387	–	pass
388
389	+	numEventsInBlock = self.eventsbyblock[block]
390	+	common.logger.debug(5,'Events in Block File '+str(numEventsInBlock))
391	+
392	+	files = self.filesbyblock[block]
393	+	numFilesInBlock = len(files)
394	+	if (numFilesInBlock <= 0):
395	+	continue
396	+	fileCount = 0
397	+
398	+	# ---- New block => New job ---- #
399	+	parString = "\\{"
400	+	# counter for number of events in files currently worked on
401	+	filesEventCount = 0
402	+	# flag if next while loop should touch new file
403	+	newFile = 1
404	+	# job event counter
405	+	jobSkipEventCount = 0
406	+
407	+	# ---- Iterate over the files in the block until we've met the requested ---- #
408	+	# ---- total # of events or we've gone over all the files in this block  ---- #
409	+	while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) and (jobCount < totalNumberOfJobs) ):
410	+	file = files[fileCount]
411	+	if newFile :
412	+	try:
413	+	numEventsInFile = self.eventsbyfile[file]
414	+	common.logger.debug(6, "File "+str(file)+" has "+str(numEventsInFile)+" events")
415	+	# increase filesEventCount
416	+	filesEventCount += numEventsInFile
417	+	# Add file to current job
418	+	parString += '\\\"' + file + '\\\"\,'
419	+	newFile = 0
420	+	except KeyError:
421	+	common.logger.message("File "+str(file)+" has unknown number of events: skipping")
422	+
423	+
424	+	# if less events in file remain than eventsPerJobRequested
425	+	if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested ) :
426	+	# if last file in block
427	+	if ( fileCount == numFilesInBlock-1 ) :
428	+	# end job using last file, use remaining events in block
429	+	# close job and touch new file
430	+	fullString = parString[:-2]
431	+	fullString += '\\}'
432	+	list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
433	+	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(filesEventCount - jobSkipEventCount)+" events (last file in block).")
434	+	self.jobDestination.append(blockSites[block])
435	+	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
436	+	# reset counter
437	+	jobCount = jobCount + 1
438	+	totalEventCount = totalEventCount + filesEventCount - jobSkipEventCount
439	+	eventsRemaining = eventsRemaining - filesEventCount + jobSkipEventCount
440	+	jobSkipEventCount = 0
441	+	# reset file
442	+	parString = "\\{"
443	+	filesEventCount = 0
444	+	newFile = 1
445	+	fileCount += 1
446	+	else :
447	+	# go to next file
448	+	newFile = 1
449	+	fileCount += 1
450	+	# if events in file equal to eventsPerJobRequested
451	+	elif ( filesEventCount - jobSkipEventCount == eventsPerJobRequested ) :
452	+	# close job and touch new file
453	+	fullString = parString[:-2]
454	+	fullString += '\\}'
455	+	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
456	+	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
457	+	self.jobDestination.append(blockSites[block])
458	+	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
459	+	# reset counter
460	+	jobCount = jobCount + 1
461	+	totalEventCount = totalEventCount + eventsPerJobRequested
462	+	eventsRemaining = eventsRemaining - eventsPerJobRequested
463	+	jobSkipEventCount = 0
464	+	# reset file
465	+	parString = "\\{"
466	+	filesEventCount = 0
467	+	newFile = 1
468	+	fileCount += 1
469	+
470	+	# if more events in file remain than eventsPerJobRequested
471	+	else :
472	+	# close job but don't touch new file
473	+	fullString = parString[:-2]
474	+	fullString += '\\}'
475	+	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
476	+	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
477	+	self.jobDestination.append(blockSites[block])
478	+	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
479	+	# increase counter
480	+	jobCount = jobCount + 1
481	+	totalEventCount = totalEventCount + eventsPerJobRequested
482	+	eventsRemaining = eventsRemaining - eventsPerJobRequested
483	+	# calculate skip events for last file
484	+	# use filesEventCount (contains several files), jobSkipEventCount and eventsPerJobRequest
485	+	jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file])
486	+	# remove all but the last file
487	+	filesEventCount = self.eventsbyfile[file]
488	+	parString = "\\{"
489	+	parString += '\\\"' + file + '\\\"\,'
490	+	pass # END if
491	+	pass # END while (iterate over files in the block)
492	+	pass # END while (iterate over blocks in the dataset)
493	+	self.ncjobs = self.total_number_of_jobs = jobCount
494	+	if (eventsRemaining > 0 and jobCount < totalNumberOfJobs ):
495	+	common.logger.message("Could not run on all requested events because some blocks not hosted at allowed sites.")
496	+	common.logger.message("\n"+str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n")
497	+
498		self.list_of_args = list_of_lists
390	–	#print self.list_of_args
499		return
500
501	<	def jobSplittingPerEvents(self):
501	>	def jobSplittingNoInput(self):
502		"""
503		Perform job splitting based on number of event per job
504		"""
505		common.logger.debug(5,'Splitting per events')
506		common.logger.message('Required '+str(self.eventsPerJob)+' events per job ')
507	+	common.logger.message('Required '+str(self.theNumberOfJobs)+' jobs in total ')
508		common.logger.message('Required '+str(self.total_number_of_events)+' events in total ')
509
510		if (self.total_number_of_events < 0):
511		msg='Cannot split jobs per Events with "-1" as total number of events'
512		raise CrabException(msg)
513
514	<	self.total_number_of_jobs = int(self.total_number_of_events/self.eventsPerJob)
514	>	if (self.selectEventsPerJob):
515	>	self.total_number_of_jobs = int(self.total_number_of_events/self.eventsPerJob)
516	>	elif (self.selectNumberOfJobs) :
517	>	self.total_number_of_jobs = self.theNumberOfJobs
518	>	self.eventsPerJob = int(self.total_number_of_events/self.total_number_of_jobs)
519
407	–	print "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"
408	–	print "self.total_number_of_events = ", self.total_number_of_events
409	–	print "self.eventsPerJob = ", self.eventsPerJob
410	–	print "self.total_number_of_jobs = ", self.total_number_of_jobs
411	–	print "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"
412	–
520		common.logger.debug(5,'N jobs  '+str(self.total_number_of_jobs))
521
522		# is there any remainder?
#	Line 417 | Line 524 | class Cmssw(JobType):
524
525		common.logger.debug(5,'Check  '+str(check))
526
527	+	common.logger.message(str(self.total_number_of_jobs)+' jobs can be created, each for '+str(self.eventsPerJob)+' for a total of '+str(self.total_number_of_jobs*self.eventsPerJob)+' events')
528		if check > 0:
529	<	common.logger.message('Warning: asked '+self.total_number_of_events+' but will do only '+(int(self.total_number_of_jobs)*self.eventsPerJob))
422	<
423	<	common.logger.message(str(self.total_number_of_jobs)+' jobs will be created for a total of '+str(self.total_number_of_jobs*self.eventsPerJob)+' events')
529	>	common.logger.message('Warning: asked '+str(self.total_number_of_events)+' but can do only '+str(int(self.total_number_of_jobs)*self.eventsPerJob))
530
531		# argument is seed number.$i
532		self.list_of_args = []
533		for i in range(self.total_number_of_jobs):
534	<	self.list_of_args.append(int(str(self.sourceSeed)+str(i)))
535	<	print self.list_of_args
534	>	## Since there is no input, any site is good
535	>	# self.jobDestination.append(["Any"])
536	>	self.jobDestination.append([""]) #must be empty to write correctly the xml
537	>	if (self.sourceSeed):
538	>	if (self.sourceSeedVtx):
539	>	## pythia + vtx random seed
540	>	self.list_of_args.append([
541	>	str(self.sourceSeed)+str(i),
542	>	str(self.sourceSeedVtx)+str(i)
543	>	])
544	>	else:
545	>	## only pythia random seed
546	>	self.list_of_args.append([(str(self.sourceSeed)+str(i))])
547	>	else:
548	>	## no random seed
549	>	self.list_of_args.append([str(i)])
550	>	#print self.list_of_args
551
552		return
553
554	+
555	+	def jobSplittingForScript(self):#CarlosDaniele
556	+	"""
557	+	Perform job splitting based on number of job
558	+	"""
559	+	common.logger.debug(5,'Splitting per job')
560	+	common.logger.message('Required '+str(self.theNumberOfJobs)+' jobs in total ')
561	+
562	+	self.total_number_of_jobs = self.theNumberOfJobs
563	+
564	+	common.logger.debug(5,'N jobs  '+str(self.total_number_of_jobs))
565	+
566	+	common.logger.message(str(self.total_number_of_jobs)+' jobs can be created')
567	+
568	+	# argument is seed number.$i
569	+	self.list_of_args = []
570	+	for i in range(self.total_number_of_jobs):
571	+	## Since there is no input, any site is good
572	+	# self.jobDestination.append(["Any"])
573	+	self.jobDestination.append([""])
574	+	## no random seed
575	+	self.list_of_args.append([str(i)])
576	+	return
577	+
578		def split(self, jobParams):
579
580		common.jobDB.load()
#	Line 441 | Line 586 | class Cmssw(JobType):
586		jobParams.append("")
587
588		for job in range(njobs):
589	<	jobParams[job] = str(arglist[job])
589	>	jobParams[job] = arglist[job]
590	>	# print str(arglist[job])
591	>	# print jobParams[job]
592		common.jobDB.setArguments(job, jobParams[job])
593	+	common.logger.debug(5,"Job "+str(job)+" Destination: "+str(self.jobDestination[job]))
594	+	common.jobDB.setDestination(job, self.jobDestination[job])
595
596		common.jobDB.save()
597		return
598
599		def getJobTypeArguments(self, nj, sched):
600	<	return common.jobDB.arguments(nj)
600	>	result = ''
601	>	for i in common.jobDB.arguments(nj):
602	>	result=result+str(i)+" "
603	>	return result
604
605		def numberOfJobs(self):
606		# Fabio
607		return self.total_number_of_jobs
608
457	–	def checkBlackList(self, allSites):
458	–	if len(self.reCEBlackList)==0: return allSites
459	–	sites = []
460	–	for site in allSites:
461	–	common.logger.debug(10,'Site '+site)
462	–	good=1
463	–	for re in self.reCEBlackList:
464	–	if re.search(site):
465	–	common.logger.message('CE in black list, skipping site '+site)
466	–	good=0
467	–	pass
468	–	if good: sites.append(site)
469	–	if len(sites) == 0:
470	–	common.logger.debug(3,"No sites found after BlackList")
471	–	return sites
472	–
473	–	def checkWhiteList(self, allSites):
474	–
475	–	if len(self.reCEWhiteList)==0: return allSites
476	–	sites = []
477	–	for site in allSites:
478	–	good=0
479	–	for re in self.reCEWhiteList:
480	–	if re.search(site):
481	–	common.logger.debug(5,'CE in white list, adding site '+site)
482	–	good=1
483	–	if not good: continue
484	–	sites.append(site)
485	–	if len(sites) == 0:
486	–	common.logger.message("No sites found after WhiteList\n")
487	–	else:
488	–	common.logger.debug(5,"Selected sites via WhiteList are "+str(sites)+"\n")
489	–	return sites
490	–
609		def getTarBall(self, exe):
610		"""
611		Return the TarBall with lib and exe
#	Line 553 | Line 671 | class Cmssw(JobType):
671		dataDir = 'src/Data/'
672		if os.path.isdir(swArea+'/'+dataDir):
673		filesToBeTarred.append(dataDir)
674	<
674	>
675	>	## copy ProdAgent dir to swArea
676	>	cmd = '\cp -rf ' + os.environ['CRABDIR'] + '/ProdAgentApi ' + swArea
677	>	cmd_out = runCommand(cmd)
678	>	if cmd_out != '':
679	>	common.logger.message('ProdAgentApi directory could not be copied to local CMSSW project directory.')
680	>	common.logger.message('No FrameworkJobreport parsing is possible on the WorkerNode.')
681	>
682	>	## Now check if the Data dir is present
683	>	paDir = 'ProdAgentApi'
684	>	if os.path.isdir(swArea+'/'+paDir):
685	>	filesToBeTarred.append(paDir)
686	>
687		## Create the tar-ball
688		if len(filesToBeTarred)>0:
689		cwd = os.getcwd()
#	Line 583 | Line 713 | class Cmssw(JobType):
713		txt += 'if [ $middleware == LCG ]; then \n'
714		txt += self.wsSetupCMSLCGEnvironment_()
715		txt += 'elif [ $middleware == OSG ]; then\n'
716	<	txt += '    time=`date -u +"%s"`\n'
717	<	txt += '    WORKING_DIR=$OSG_WN_TMP/cms_$time\n'
588	<	txt += '    echo "Creating working directory: $WORKING_DIR"\n'
589	<	txt += '    /bin/mkdir -p $WORKING_DIR\n'
716	>	txt += '    WORKING_DIR=`/bin/mktemp  -d $OSG_WN_TMP/cms_XXXXXXXXXXXX`\n'
717	>	txt += '    echo "Created working directory: $WORKING_DIR"\n'
718		txt += '    if [ ! -d $WORKING_DIR ] ;then\n'
719		txt += '        echo "SET_CMS_ENV 10016 ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
720		txt += '        echo "JOB_EXIT_STATUS = 10016"\n'
#	Line 643 | Line 771 | class Cmssw(JobType):
771		txt += "\n"
772		txt += "## number of arguments (first argument always jobnumber)\n"
773		txt += "\n"
774	<	txt += "narg=$#\n"
775	<	txt += "if [ $narg -lt 2 ]\n"
774	>	#        txt += "narg=$#\n"
775	>	txt += "if [ $nargs -lt 2 ]\n"
776		txt += "then\n"
777	<	txt += "    echo 'SET_EXE_ENV 1 ==> ERROR Too few arguments' +$narg+ \n"
777	>	txt += "    echo 'SET_EXE_ENV 1 ==> ERROR Too few arguments' +$nargs+ \n"
778		txt += '    echo "JOB_EXIT_STATUS = 50113"\n'
779		txt += '    echo "JobExitCode=50113" | tee -a $RUNTIME_AREA/$repo\n'
780		txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
#	Line 674 | Line 802 | class Cmssw(JobType):
802
803		# Prepare job-specific part
804		job = common.job_list[nj]
805	<	pset = os.path.basename(job.configFilename())
806	<	txt += '\n'
807	<	if (self.datasetPath): # standard job
808	<	txt += 'InputFiles=$2\n'
809	<	txt += 'echo "Inputfiles:<$InputFiles>"\n'
810	<	txt += 'sed "s#{\'INPUT\'}#$InputFiles#" $RUNTIME_AREA/'+pset+' > pset.cfg\n'
811	<	else:  # pythia like job
812	<	txt += 'Seed=$2\n'
813	<	txt += 'echo "Seed: <$Seed>"\n'
814	<	txt += 'sed "s#INPUT#$Seed#" $RUNTIME_AREA/'+pset+' > pset.cfg\n'
805	>	if self.pset != None: #CarlosDaniele
806	>	pset = os.path.basename(job.configFilename())
807	>	txt += '\n'
808	>	if (self.datasetPath): # standard job
809	>	#txt += 'InputFiles=$2\n'
810	>	txt += 'InputFiles=${args[1]}\n'
811	>	txt += 'MaxEvents=${args[2]}\n'
812	>	txt += 'SkipEvents=${args[3]}\n'
813	>	txt += 'echo "Inputfiles:<$InputFiles>"\n'
814	>	txt += 'sed "s#{\'INPUT\'}#$InputFiles#" $RUNTIME_AREA/'+pset+' > pset_tmp_1.cfg\n'
815	>	txt += 'echo "MaxEvents:<$MaxEvents>"\n'
816	>	txt += 'sed "s#INPUTMAXEVENTS#$MaxEvents#" pset_tmp_1.cfg > pset_tmp_2.cfg\n'
817	>	txt += 'echo "SkipEvents:<$SkipEvents>"\n'
818	>	txt += 'sed "s#INPUTSKIPEVENTS#$SkipEvents#" pset_tmp_2.cfg > pset.cfg\n'
819	>	else:  # pythia like job
820	>	if (self.sourceSeed):
821	>	#                    txt += 'Seed=$2\n'
822	>	txt += 'Seed=${args[1]}\n'
823	>	txt += 'echo "Seed: <$Seed>"\n'
824	>	txt += 'sed "s#\<INPUT\>#$Seed#" $RUNTIME_AREA/'+pset+' > tmp.cfg\n'
825	>	if (self.sourceSeedVtx):
826	>	#                        txt += 'VtxSeed=$3\n'
827	>	txt += 'VtxSeed=${args[2]}\n'
828	>	txt += 'echo "VtxSeed: <$VtxSeed>"\n'
829	>	txt += 'sed "s#INPUTVTX#$VtxSeed#" tmp.cfg > pset.cfg\n'
830	>	else:
831	>	txt += 'mv tmp.cfg pset.cfg\n'
832	>	else:
833	>	txt += '# Copy untouched pset\n'
834	>	txt += 'cp $RUNTIME_AREA/'+pset+' pset.cfg\n'
835	>
836
837		if len(self.additional_inbox_files) > 0:
838		for file in self.additional_inbox_files:
839	<	txt += 'if [ -e $RUNTIME_AREA/'+file+' ] ; then\n'
840	<	txt += '   cp $RUNTIME_AREA/'+file+' .\n'
841	<	txt += '   chmod +x '+file+'\n'
839	>	relFile = file.split("/")[-1]
840	>	txt += 'if [ -e $RUNTIME_AREA/'+relFile+' ] ; then\n'
841	>	txt += '   cp $RUNTIME_AREA/'+relFile+' .\n'
842	>	txt += '   chmod +x '+relFile+'\n'
843		txt += 'fi\n'
844		pass
845
846	<	txt += 'echo "### END JOB SETUP ENVIRONMENT ###"\n\n'
847	<
848	<	txt += '\n'
849	<	txt += 'echo "***** cat pset.cfg *********"\n'
850	<	txt += 'cat pset.cfg\n'
851	<	txt += 'echo "****** end pset.cfg ********"\n'
852	<	txt += '\n'
853	<	# txt += 'echo "***** cat pset1.cfg *********"\n'
854	<	# txt += 'cat pset1.cfg\n'
855	<	# txt += 'echo "****** end pset1.cfg ********"\n'
846	>	if self.pset != None: #CarlosDaniele
847	>	txt += 'echo "### END JOB SETUP ENVIRONMENT ###"\n\n'
848	>
849	>	txt += '\n'
850	>	txt += 'echo "***** cat pset.cfg *********"\n'
851	>	txt += 'cat pset.cfg\n'
852	>	txt += 'echo "****** end pset.cfg ********"\n'
853	>	txt += '\n'
854	>	# txt += 'echo "***** cat pset1.cfg *********"\n'
855	>	# txt += 'cat pset1.cfg\n'
856	>	# txt += 'echo "****** end pset1.cfg ********"\n'
857		return txt
858
859		def wsBuildExe(self, nj):
#	Line 740 | Line 891 | class Cmssw(JobType):
891		txt += 'else \n'
892		txt += '   echo "Successful untar" \n'
893		txt += 'fi \n'
894	+	txt += '\n'
895	+	txt += 'echo "Include ProdAgentApi in PYTHONPATH"\n'
896	+	txt += 'if [ -z "$PYTHONPATH" ]; then\n'
897	+	txt += '   export PYTHONPATH=ProdAgentApi\n'
898	+	txt += 'else\n'
899	+	txt += '   export PYTHONPATH=ProdAgentApi:${PYTHONPATH}\n'
900	+	txt += 'fi\n'
901	+	txt += '\n'
902	+
903		pass
904
905		return txt
#	Line 751 | Line 911 | class Cmssw(JobType):
911		"""
912
913		def executableName(self):
914	<	return self.executable
914	>	if self.pset == None: #CarlosDaniele
915	>	return "sh "
916	>	else:
917	>	return self.executable
918
919		def executableArgs(self):
920	<	return " -p pset.cfg"
920	>	if self.pset == None:#CarlosDaniele
921	>	return   self.scriptExe + " $NJob"
922	>	else:
923	>	return " -p pset.cfg"
924
925		def inputSandbox(self, nj):
926		"""
927		Returns a list of filenames to be put in JDL input sandbox.
928		"""
929		inp_box = []
930	<	# dict added to delete duplicate from input sandbox file list
931	<	seen = {}
930	>	# # dict added to delete duplicate from input sandbox file list
931	>	# seen = {}
932		## code
933		if os.path.isfile(self.tgzNameWithPath):
934		inp_box.append(self.tgzNameWithPath)
935		## config
936	<	inp_box.append(common.job_list[nj].configFilename())
936	>	if not self.pset is None: #CarlosDaniele
937	>	inp_box.append(common.job_list[nj].configFilename())
938		## additional input files
939		#for file in self.additional_inbox_files:
940		#    inp_box.append(common.work_space.cwdDir()+file)
#	Line 779 | Line 946 | class Cmssw(JobType):
946		"""
947		out_box = []
948
782	–	stdout=common.job_list[nj].stdout()
783	–	stderr=common.job_list[nj].stderr()
784	–
949		## User Declared output files
950	<	for out in self.output_file:
950	>	for out in (self.output_file+self.output_file_sandbox):
951		n_out = nj + 1
952		out_box.append(self.numberFile_(out,str(n_out)))
953		return out_box
790	–	return []
954
955		def prepareSteeringCards(self):
956		"""
#	Line 803 | Line 966 | class Cmssw(JobType):
966		txt = '\n'
967		txt += '# directory content\n'
968		txt += 'ls \n'
969	<	file_list = ''
970	<	for fileWithSuffix in self.output_file:
969	>
970	>	for fileWithSuffix in (self.output_file+self.output_file_sandbox):
971		output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
809	–	file_list=file_list+output_file_num+' '
972		txt += '\n'
973		txt += '# check output file\n'
974		txt += 'ls '+fileWithSuffix+'\n'
975	<	txt += 'exe_result=$?\n'
976	<	txt += 'if [ $exe_result -ne 0 ] ; then\n'
977	<	txt += '   echo "ERROR: No output file to manage"\n'
816	<	txt += '   echo "JOB_EXIT_STATUS = $exe_result"\n'
817	<	txt += '   echo "JobExitCode=60302" | tee -a $RUNTIME_AREA/$repo\n'
818	<	txt += '   dumpStatus $RUNTIME_AREA/$repo\n'
819	<	txt += '   rm -f $RUNTIME_AREA/$repo \n'
820	<	txt += '   echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
821	<	txt += '   echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
822	<	### OLI_DANIELE
975	>	txt += 'ls_result=$?\n'
976	>	txt += 'if [ $ls_result -ne 0 ] ; then\n'
977	>	txt += '   echo "ERROR: Problem with output file"\n'
978		if common.scheduler.boss_scheduler_name == 'condor_g':
979		txt += '    if [ $middleware == OSG ]; then \n'
980		txt += '        echo "prepare dummy output file"\n'
#	Line 830 | Line 985 | class Cmssw(JobType):
985		txt += 'fi\n'
986
987		txt += 'cd $RUNTIME_AREA\n'
988	<	file_list=file_list[:-1]
834	<	txt += 'file_list="'+file_list+'"\n'
988	>	txt += 'cd $RUNTIME_AREA\n'
989		### OLI_DANIELE
990		txt += 'if [ $middleware == OSG ]; then\n'
991		txt += '    cd $RUNTIME_AREA\n'
#	Line 848 | Line 1002 | class Cmssw(JobType):
1002		txt += '    fi\n'
1003		txt += 'fi\n'
1004		txt += '\n'
1005	+
1006	+	file_list = ''
1007	+	## Add to filelist only files to be possibly copied to SE
1008	+	for fileWithSuffix in self.output_file:
1009	+	output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1010	+	file_list=file_list+output_file_num+' '
1011	+	file_list=file_list[:-1]
1012	+	txt += 'file_list="'+file_list+'"\n'
1013	+
1014		return txt
1015
1016		def numberFile_(self, file, txt):
#	Line 862 | Line 1025 | class Cmssw(JobType):
1025		# add "_txt"
1026		if len(p)>1:
1027		ext = p[len(p)-1]
865	–	#result = name + '_' + str(txt) + "." + ext
1028		result = name + '_' + txt + "." + ext
1029		else:
868	–	#result = name + '_' + str(txt)
1030		result = name + '_' + txt
1031
1032		return result
#	Line 875 | Line 1036 | class Cmssw(JobType):
1036		return job requirements to add to jdl files
1037		"""
1038		req = ''
1039	<	if common.analisys_common_info['sw_version']:
1039	>	if self.version:
1040		req='Member("VO-cms-' + \
1041	<	common.analisys_common_info['sw_version'] + \
1041	>	self.version + \
1042		'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1043	<	if common.analisys_common_info['sites']:
1044	<	if len(common.analisys_common_info['sites'])>0:
1045	<	req = req + ' && ('
885	<	for i in range(len(common.analisys_common_info['sites'])):
886	<	req = req + 'other.GlueCEInfoHostName == "' \
887	<	+ common.analisys_common_info['sites'][i] + '"'
888	<	if ( i < (int(len(common.analisys_common_info['sites']) - 1)) ):
889	<	req = req + ' || '
890	<	req = req + ')'
891	<	#print "req = ", req
1043	>
1044	>	req = req + ' && (other.GlueHostNetworkAdapterOutboundIP)'
1045	>
1046		return req
1047
1048		def configFilename(self):
#	Line 906 | Line 1060 | class Cmssw(JobType):
1060		txt += '   if [ -f $GRID3_APP_DIR/cmssoft/cmsset_default.sh ] ;then\n'
1061		txt += '      # Use $GRID3_APP_DIR/cmssoft/cmsset_default.sh to setup cms software\n'
1062		txt += '       source $GRID3_APP_DIR/cmssoft/cmsset_default.sh '+self.version+'\n'
1063	<	txt += '   elif [ -f $OSG_APP/cmssoft/cmsset_default.sh ] ;then\n'
1064	<	txt += '      # Use $OSG_APP/cmssoft/cmsset_default.sh to setup cms software\n'
1065	<	txt += '       source $OSG_APP/cmssoft/cmsset_default.sh '+self.version+'\n'
1063	>	txt += '   elif [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n'
1064	>	txt += '      # Use $OSG_APP/cmssoft/cms/cmsset_default.sh to setup cms software\n'
1065	>	txt += '       source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1066		txt += '   else\n'
1067	<	txt += '       echo "SET_CMS_ENV 10020 ==> ERROR $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cmsset_default.sh file not found"\n'
1067	>	txt += '       echo "SET_CMS_ENV 10020 ==> ERROR $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1068		txt += '       echo "JOB_EXIT_STATUS = 10020"\n'
1069		txt += '       echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
1070		txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
#	Line 923 | Line 1077 | class Cmssw(JobType):
1077		txt += '       cd $RUNTIME_AREA\n'
1078		txt += '       /bin/rm -rf $WORKING_DIR\n'
1079		txt += '       if [ -d $WORKING_DIR ] ;then\n'
1080	<	txt += '            echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cmsset_default.sh file not found"\n'
1080	>	txt += '            echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1081		txt += '            echo "JOB_EXIT_STATUS = 10017"\n'
1082		txt += '            echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n'
1083		txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
#	Line 1016 | Line 1170 | class Cmssw(JobType):
1170
1171		def getTaskid(self):
1172		return self._taskId
1173	+
1174	+	#######################################################################
1175	+	def uniquelist(self, old):
1176	+	"""
1177	+	remove duplicates from a list
1178	+	"""
1179	+	nd={}
1180	+	for e in old:
1181	+	nd[e]=0
1182	+	return nd.keys()

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines