ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/cms_cmssw.py

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.2 by slacapra, Wed Apr 26 15:31:06 2006 UTC vs.
Revision 1.37 by mkirn, Thu Aug 10 17:08:54 2006 UTC

#	Line 2 | Line 2 | from JobType import JobType
2		from crab_logger import Logger
3		from crab_exceptions import *
4		from crab_util import *
5	+	import math
6		import common
7	+	import PsetManipulator
8
9	<	import DataDiscovery
10	<	import DataLocation
9	>	import DBSInfo_EDM
10	>	import DataDiscovery_EDM
11	>	import DataLocation_EDM
12		import Scram
13
14		import os, string, re
#	Line 16 | Line 19 | class Cmssw(JobType):
19		common.logger.debug(3,'CMSSW::__init__')
20
21		self.analisys_common_info = {}
22	<
22	>	# Marco.
23	>	self._params = {}
24	>	self.cfg_params = cfg_params
25		log = common.logger
26
27		self.scram = Scram.Scram(cfg_params)
#	Line 26 | Line 31 | class Cmssw(JobType):
31		self.executable = ''
32		self.tgz_name = 'default.tgz'
33
34	+
35		self.version = self.scram.getSWVersion()
36	+	self.setParam_('application', self.version)
37		common.analisys_common_info['sw_version'] = self.version
38	+	### FEDE
39	+	common.analisys_common_info['copy_input_data'] = 0
40	+	common.analisys_common_info['events_management'] = 1
41
42		### collect Data cards
43		try:
44	<	self.owner = cfg_params['CMSSW.owner']
45	<	log.debug(6, "CMSSW::CMSSW(): owner = "+self.owner)
46	<	self.dataset = cfg_params['CMSSW.dataset']
47	<	log.debug(6, "CMSSW::CMSSW(): dataset = "+self.dataset)
44	>	tmp =  cfg_params['CMSSW.datasetpath']
45	>	log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
46	>	if string.lower(tmp)=='none':
47	>	self.datasetPath = None
48	>	self.selectNoInput = 1
49	>	else:
50	>	self.datasetPath = tmp
51	>	self.selectNoInput = 0
52		except KeyError:
53	<	msg = "Error: owner and/or dataset not defined "
53	>	msg = "Error: datasetpath not defined "
54		raise CrabException(msg)
55
56	+	# ML monitoring
57	+	# split dataset path style: /PreProdR3Minbias/SIM/GEN-SIM
58	+	if not self.datasetPath:
59	+	self.setParam_('dataset', 'None')
60	+	self.setParam_('owner', 'None')
61	+	else:
62	+	datasetpath_split = self.datasetPath.split("/")
63	+	self.setParam_('dataset', datasetpath_split[1])
64	+	self.setParam_('owner', datasetpath_split[-1])
65	+
66	+	self.setTaskid_()
67	+	self.setParam_('taskId', self.cfg_params['taskId'])
68	+
69		self.dataTiers = []
43	–	try:
44	–	tmpDataTiers = string.split(cfg_params['CMSSW.data_tier'],',')
45	–	for tmp in tmpDataTiers:
46	–	tmp=string.strip(tmp)
47	–	self.dataTiers.append(tmp)
48	–	pass
49	–	pass
50	–	except KeyError:
51	–	pass
52	–	log.debug(6, "Cmssw::Cmssw(): dataTiers = "+str(self.dataTiers))
70
71		## now the application
72		try:
73		self.executable = cfg_params['CMSSW.executable']
74	+	self.setParam_('exe', self.executable)
75		log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable)
76		msg = "Default executable cmsRun overridden. Switch to " + self.executable
77		log.debug(3,msg)
78		except KeyError:
79		self.executable = 'cmsRun'
80	+	self.setParam_('exe', self.executable)
81		msg = "User executable not defined. Use cmsRun"
82		log.debug(3,msg)
83		pass
#	Line 83 | Line 102 | class Cmssw(JobType):
102		tmp=string.strip(tmp)
103		self.output_file.append(tmp)
104		pass
86	–
105		else:
106		log.message("No output file defined: only stdout/err will be available")
107		pass
#	Line 94 | Line 112 | class Cmssw(JobType):
112
113		# script_exe file as additional file in inputSandbox
114		try:
115	<	self.scriptExe = cfg_params['CMSSW.script_exe']
116	<	self.additional_inbox_files.append(self.scriptExe)
115	>	self.scriptExe = cfg_params['USER.script_exe']
116	>	self.additional_inbox_files.append(self.scriptExe)
117	>	if self.scriptExe != '':
118	>	if not os.path.isfile(self.scriptExe):
119	>	msg ="WARNING. file "+self.scriptExe+" not found"
120	>	raise CrabException(msg)
121		except KeyError:
122		pass
101	–	if self.scriptExe != '':
102	–	if os.path.isfile(self.scriptExe):
103	–	pass
104	–	else:
105	–	log.message("WARNING. file "+self.scriptExe+" not found")
106	–	sys.exit()
123
124		## additional input files
125		try:
126	<	tmpAddFiles = string.split(cfg_params['CMSSW.additional_input_files'],',')
126	>	tmpAddFiles = string.split(cfg_params['USER.additional_input_files'],',')
127		for tmp in tmpAddFiles:
128	<	tmp=string.strip(tmp)
129	<	self.additional_inbox_files.append(tmp)
128	>	if not os.path.exists(tmp):
129	>	raise CrabException("Additional input file not found: "+tmp)
130	>	self.additional_inbox_files.append(string.strip(tmp))
131		pass
132		pass
133		except KeyError:
134		pass
135
136	+	# files per job
137	+	try:
138	+	if (cfg_params['CMSSW.files_per_jobs']):
139	+	raise CrabException("files_per_jobs no longer supported.  Quitting.")
140	+	except KeyError:
141	+	pass
142	+
143	+	## Events per job
144	+	try:
145	+	self.eventsPerJob =int( cfg_params['CMSSW.events_per_job'])
146	+	self.selectEventsPerJob = 1
147	+	except KeyError:
148	+	self.eventsPerJob = -1
149	+	self.selectEventsPerJob = 0
150	+
151	+	## number of jobs
152	+	try:
153	+	self.theNumberOfJobs =int( cfg_params['CMSSW.number_of_jobs'])
154	+	self.selectNumberOfJobs = 1
155	+	except KeyError:
156	+	self.theNumberOfJobs = 0
157	+	self.selectNumberOfJobs = 0
158	+
159		try:
160		self.total_number_of_events = int(cfg_params['CMSSW.total_number_of_events'])
161	+	self.selectTotalNumberEvents = 1
162		except KeyError:
163	<	msg = 'Must define total_number_of_events and job_number_of_events'
163	>	self.total_number_of_events = 0
164	>	self.selectTotalNumberEvents = 0
165	>
166	>	if ( (self.selectTotalNumberEvents + self.selectEventsPerJob + self.selectNumberOfJobs) != 2 ):
167	>	msg = 'Must define exactly two of total_number_of_events, events_per_job, or number_of_jobs.'
168		raise CrabException(msg)
124	–
125	–	#Marco: FirstEvent is nolonger used inside PSet
126	–	#        try:
127	–	#            self.first = int(cfg_params['CMSSW.first_event'])
128	–	#        except KeyError:
129	–	#            self.first = 0
130	–	#            pass
131	–	#        log.debug(6, "Orca::Orca(): total number of events = "+`self.total_number_of_events`)
132	–	#log.debug(6, "Orca::Orca(): events per job = "+`self.job_number_of_events`)
133	–	#        log.debug(6, "Orca::Orca(): first event = "+`self.first`)
134	–
135	–	CEBlackList = []
136	–	try:
137	–	tmpBad = string.split(cfg_params['EDG.ce_black_list'],',')
138	–	for tmp in tmpBad:
139	–	tmp=string.strip(tmp)
140	–	CEBlackList.append(tmp)
141	–	except KeyError:
142	–	pass
169
170	<	self.reCEBlackList=[]
171	<	for bad in CEBlackList:
172	<	self.reCEBlackList.append(re.compile( bad ))
147	<
148	<	common.logger.debug(5,'CEBlackList: '+str(CEBlackList))
149	<
150	<	CEWhiteList = []
151	<	try:
152	<	tmpGood = string.split(cfg_params['EDG.ce_white_list'],',')
153	<	#tmpGood = ['cern']
154	<	for tmp in tmpGood:
155	<	tmp=string.strip(tmp)
156	<	#if (tmp == 'cnaf'): tmp = 'webserver' ########## warning: temp. patch
157	<	CEWhiteList.append(tmp)
170	>	## source seed for pythia
171	>	try:
172	>	self.sourceSeed = int(cfg_params['CMSSW.pythia_seed'])
173		except KeyError:
174	<	pass
174	>	self.sourceSeed = None
175	>	common.logger.debug(5,"No seed given")
176
177	<	#print 'CEWhiteList: ',CEWhiteList
178	<	self.reCEWhiteList=[]
179	<	for Good in CEWhiteList:
180	<	self.reCEWhiteList.append(re.compile( Good ))
177	>	try:
178	>	self.sourceSeedVtx = int(cfg_params['CMSSW.vtx_seed'])
179	>	except KeyError:
180	>	self.sourceSeedVtx = None
181	>	common.logger.debug(5,"No vertex seed given")
182
183	<	common.logger.debug(5,'CEWhiteList: '+str(CEWhiteList))
183	>	self.PsetEdit = PsetManipulator.PsetManipulator(self.pset) #Daniele Pset
184
185		#DBSDLS-start
186		## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
187		self.maxEvents=0  # max events available   ( --> check the requested nb. of evts in Creator.py)
188		self.DBSPaths={}  # all dbs paths requested ( --> input to the site local discovery script)
189	+	self.jobDestination=[]  # Site destination(s) for each job (list of lists)
190		## Perform the data location and discovery (based on DBS/DLS)
191	<	self.DataDiscoveryAndLocation(cfg_params)
191	>	## SL: Don't if NONE is specified as input (pythia use case)
192	>	blockSites = {}
193	>	if self.datasetPath:
194	>	blockSites = self.DataDiscoveryAndLocation(cfg_params)
195		#DBSDLS-end
196
197		self.tgzNameWithPath = self.getTarBall(self.executable)
198	+
199	+	## Select Splitting
200	+	if self.selectNoInput: self.jobSplittingNoInput()
201	+	else: self.jobSplittingByBlocks(blockSites)
202	+
203	+	# modify Pset
204	+	try:
205	+	if (self.datasetPath): # standard job
206	+	# always process all events in a file
207	+	self.PsetEdit.maxEvent("-1")
208	+	self.PsetEdit.inputModule("INPUT")
209	+
210	+	else:  # pythia like job
211	+	self.PsetEdit.maxEvent(self.eventsPerJob)
212	+	if (self.sourceSeed) :
213	+	self.PsetEdit.pythiaSeed("INPUT")
214	+	if (self.sourceSeedVtx) :
215	+	self.PsetEdit.pythiaSeedVtx("INPUTVTX")
216	+	self.PsetEdit.psetWriter(self.configFilename())
217	+	except:
218	+	msg='Error while manipuliating ParameterSet: exiting...'
219	+	raise CrabException(msg)
220
221		def DataDiscoveryAndLocation(self, cfg_params):
222
223	<	fun = "CMSSW::DataDiscoveryAndLocation()"
223	>	common.logger.debug(10,"CMSSW::DataDiscoveryAndLocation()")
224	>
225	>	datasetPath=self.datasetPath
226	>
227	>	## TODO
228	>	dataTiersList = ""
229	>	dataTiers = dataTiersList.split(',')
230
231		## Contact the DBS
232		try:
233	<	self.pubdata=DataDiscovery.DataDiscovery(self.owner,
185	<	self.dataset,
186	<	self.dataTiers,
187	<	cfg_params)
233	>	self.pubdata=DataDiscovery_EDM.DataDiscovery_EDM(datasetPath, dataTiers, cfg_params)
234		self.pubdata.fetchDBSInfo()
235
236	<	except DataDiscovery.NotExistingDatasetError, ex :
236	>	except DataDiscovery_EDM.NotExistingDatasetError, ex :
237		msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
238		raise CrabException(msg)
239
240	<	except DataDiscovery.NoDataTierinProvenanceError, ex :
240	>	except DataDiscovery_EDM.NoDataTierinProvenanceError, ex :
241		msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
242		raise CrabException(msg)
243	<	except DataDiscovery.DataDiscoveryError, ex:
243	>	except DataDiscovery_EDM.DataDiscoveryError, ex:
244		msg = 'ERROR ***: failed Data Discovery in DBS  %s'%ex.getErrorMessage()
245		raise CrabException(msg)
246
247		## get list of all required data in the form of dbs paths  (dbs path = /dataset/datatier/owner)
248	<	self.DBSPaths=self.pubdata.getDBSPaths()
249	<	common.logger.message("Required data are : ")
250	<	for path in self.DBSPaths:
251	<	common.logger.message(" --> "+path )
248	>	## self.DBSPaths=self.pubdata.getDBSPaths()
249	>	common.logger.message("Required data are :"+self.datasetPath)
250	>
251	>	self.filesbyblock=self.pubdata.getFiles()
252	>	self.eventsbyblock=self.pubdata.getEventsPerBlock()
253	>	self.eventsbyfile=self.pubdata.getEventsPerFile()
254
255		## get max number of events
208	–	common.logger.debug(10,"number of events for primary fileblocks %i"%self.pubdata.getMaxEvents())
256		self.maxEvents=self.pubdata.getMaxEvents() ##  self.maxEvents used in Creator.py
257		common.logger.message("\nThe number of available events is %s"%self.maxEvents)
258
212	–	## get fileblocks corresponding to the required data
213	–	fb=self.pubdata.getFileBlocks()
214	–	common.logger.debug(5,"fileblocks are %s"%fb)
215	–
259		## Contact the DLS and build a list of sites hosting the fileblocks
260		try:
261	<	dataloc=DataLocation.DataLocation(self.pubdata.getFileBlocks(),cfg_params)
261	>	dataloc=DataLocation_EDM.DataLocation_EDM(self.filesbyblock.keys(),cfg_params)
262		dataloc.fetchDLSInfo()
263	<	except DataLocation.DataLocationError , ex:
263	>	except DataLocation_EDM.DataLocationError , ex:
264		msg = 'ERROR ***: failed Data Location in DLS \n %s '%ex.getErrorMessage()
265		raise CrabException(msg)
266
224	–	allsites=dataloc.getSites()
225	–	common.logger.debug(5,"sites are %s"%allsites)
226	–	sites=self.checkBlackList(allsites)
227	–	common.logger.debug(5,"sites are (after black list) %s"%sites)
228	–	sites=self.checkWhiteList(sites)
229	–	common.logger.debug(5,"sites are (after white list) %s"%sites)
267
268	<	if len(sites)==0:
269	<	msg = 'No sites hosting all the needed data! Exiting... '
268	>	sites = dataloc.getSites()
269	>	allSites = []
270	>	listSites = sites.values()
271	>	for list in listSites:
272	>	for oneSite in list:
273	>	allSites.append(oneSite)
274	>	allSites = self.uniquelist(allSites)
275	>
276	>	common.logger.message("Sites ("+str(len(allSites))+") hosting part/all of dataset: "+str(allSites))
277	>	common.logger.debug(6, "List of Sites: "+str(allSites))
278	>	return sites
279	>
280	>	def jobSplittingByBlocks(self, blockSites):
281	>	"""
282	>	Perform job splitting. Jobs run over an integer number of files
283	>	and no more than one block.
284	>	ARGUMENT: blockSites: dictionary with blocks as keys and list of host sites as values
285	>	REQUIRES: self.selectTotalNumberEvents, self.selectEventsPerJob, self.selectNumberofJobs,
286	>	self.total_number_of_events, self.eventsPerJob, self.theNumberOfJobs,
287	>	self.maxEvents, self.filesbyblock
288	>	SETS: self.jobDestination - Site destination(s) for each job (a list of lists)
289	>	self.total_number_of_jobs - Total # of jobs
290	>	self.list_of_args - File(s) job will run on (a list of lists)
291	>	"""
292	>
293	>	# ---- Handle the possible job splitting configurations ---- #
294	>	if (self.selectTotalNumberEvents):
295	>	totalEventsRequested = self.total_number_of_events
296	>	if (self.selectEventsPerJob):
297	>	eventsPerJobRequested = self.eventsPerJob
298	>	if (self.selectNumberOfJobs):
299	>	totalEventsRequested = self.theNumberOfJobs * self.eventsPerJob
300	>
301	>	# If user requested all the events in the dataset
302	>	if (totalEventsRequested == -1):
303	>	eventsRemaining=self.maxEvents
304	>	# If user requested more events than are in the dataset
305	>	elif (totalEventsRequested > self.maxEvents):
306	>	eventsRemaining = self.maxEvents
307	>	common.logger.message("Requested "+str(self.total_number_of_events)+ " events, but only "+str(self.maxEvents)+" events are available.")
308	>	# If user requested less events than are in the dataset
309	>	else:
310	>	eventsRemaining = totalEventsRequested
311	>
312	>	# For user info at end
313	>	totalEventCount = 0
314	>
315	>	if (self.selectTotalNumberEvents and self.selectNumberOfJobs):
316	>	eventsPerJobRequested = int(eventsRemaining/self.theNumberOfJobs)
317	>
318	>	if (self.selectNumberOfJobs):
319	>	common.logger.message("May not create the exact number_of_jobs requested.")
320	>
321	>	blocks = blockSites.keys()
322	>	blockCount = 0
323	>	# Backup variable in case self.maxEvents counted events in a non-included block
324	>	numBlocksInDataset = len(blocks)
325	>
326	>	jobCount = 0
327	>	list_of_lists = []
328	>
329	>	# ---- Iterate over the blocks in the dataset until ---- #
330	>	# ---- we've met the requested total # of events    ---- #
331	>	while ( (eventsRemaining > 0) and (blockCount < numBlocksInDataset) ):
332	>	block = blocks[blockCount]
333	>
334	>
335	>	evInBlock = self.eventsbyblock[block]
336	>	common.logger.debug(5,'Events in Block File '+str(evInBlock))
337	>
338	>	#Correct - switch to this when DBS up
339	>	#numEventsInBlock = self.eventsbyblock[block]
340	>	numEventsInBlock = evInBlock
341	>
342	>	files = self.filesbyblock[block]
343	>	numFilesInBlock = len(files)
344	>	if (numFilesInBlock <= 0):
345	>	continue
346	>	fileCount = 0
347	>
348	>	# ---- New block => New job ---- #
349	>	parString = "\\{"
350	>	jobEventCount = 0
351	>
352	>	# ---- Iterate over the files in the block until we've met the requested ---- #
353	>	# ---- total # of events or we've gone over all the files in this block  ---- #
354	>	while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) ):
355	>	file = files[fileCount]
356	>	fileCount = fileCount + 1
357	>	numEventsInFile = self.eventsbyfile[file]
358	>	common.logger.debug(6, "File "+str(file)+" has "+str(numEventsInFile)+" events")
359	>	# Add file to current job
360	>	parString += '\\\"' + file + '\\\"\,'
361	>	jobEventCount = jobEventCount + numEventsInFile
362	>	totalEventCount = totalEventCount + numEventsInFile
363	>	eventsRemaining = eventsRemaining - numEventsInFile
364	>	if (jobEventCount >= eventsPerJobRequested):
365	>	# ---- This job has at least CMSSW.events_per_job => End of job ---- #
366	>	# Don't need the last \,
367	>	fullString = parString[:-2]
368	>	fullString += '\\}'
369	>	list_of_lists.append([fullString])
370	>	common.logger.message("Job "+str(jobCount+1)+" can run over "+str(jobEventCount)+" events.")
371	>
372	>	#self.jobDestination[jobCount] = blockSites[block]
373	>	self.jobDestination.append(blockSites[block])
374	>	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
375	>	if ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) ):
376	>	# ---- Still need CMSSW.total_number_of_events ---- #
377	>	# ---- and not about to jump into a new block  ---- #
378	>	# ---- => New job                              ---- #
379	>	parString = "\\{"
380	>	jobEventCount = 0
381	>	jobCount = jobCount + 1
382	>	pass # END if
383	>	pass # END if
384	>	pass # END while (iterate over files in the block)
385	>	if (jobEventCount < eventsPerJobRequested):
386	>	# ---- Job ending prematurely due to end of block => End of job ---- #
387	>	# Don't need the last \,
388	>	fullString = parString[:-2]
389	>	fullString += '\\}'
390	>	list_of_lists.append([fullString])
391	>	common.logger.message("Job "+str(jobCount+1)+" can run over "+str(jobEventCount)+" events.")
392	>	#self.jobDestination[jobCount] = blockSites[block]
393	>	self.jobDestination.append(blockSites[block])
394	>	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
395	>	pass # END if
396	>	blockCount = blockCount + 1
397	>	jobCount = jobCount + 1
398	>	pass # END while (iterate over blocks in the dataset)
399	>	self.total_number_of_jobs = jobCount
400	>	if (eventsRemaining > 0):
401	>	common.logger.message("Could not run on all requested events because some blocks not hosted at allowed sites.")
402	>	common.logger.message("\n"+str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n")
403	>
404	>	self.list_of_args = list_of_lists
405	>	return
406	>
407	>	def jobSplittingNoInput(self):
408	>	"""
409	>	Perform job splitting based on number of event per job
410	>	"""
411	>	common.logger.debug(5,'Splitting per events')
412	>	common.logger.message('Required '+str(self.eventsPerJob)+' events per job ')
413	>	common.logger.message('Required '+str(self.theNumberOfJobs)+' jobs in total ')
414	>	common.logger.message('Required '+str(self.total_number_of_events)+' events in total ')
415	>
416	>	if (self.total_number_of_events < 0):
417	>	msg='Cannot split jobs per Events with "-1" as total number of events'
418		raise CrabException(msg)
419	<	common.logger.message("List of Sites hosting the data : "+str(sites))
420	<	common.logger.debug(6, "List of Sites: "+str(sites))
421	<	common.analisys_common_info['sites']=sites    ## used in SchedulerEdg.py in createSchScript
419	>
420	>	if (self.selectEventsPerJob):
421	>	self.total_number_of_jobs = int(self.total_number_of_events/self.eventsPerJob)
422	>	elif (self.selectNumberOfJobs) :
423	>	self.total_number_of_jobs = self.theNumberOfJobs
424	>	self.eventsPerJob = int(self.total_number_of_events/self.total_number_of_jobs)
425	>
426	>	common.logger.debug(5,'N jobs  '+str(self.total_number_of_jobs))
427	>
428	>	# is there any remainder?
429	>	check = int(self.total_number_of_events) - (int(self.total_number_of_jobs)*self.eventsPerJob)
430	>
431	>	common.logger.debug(5,'Check  '+str(check))
432	>
433	>	common.logger.message(str(self.total_number_of_jobs)+' jobs can be created, each for '+str(self.eventsPerJob)+' for a total of '+str(self.total_number_of_jobs*self.eventsPerJob)+' events')
434	>	if check > 0:
435	>	common.logger.message('Warning: asked '+str(self.total_number_of_events)+' but can do only '+str(int(self.total_number_of_jobs)*self.eventsPerJob))
436	>
437	>	# argument is seed number.$i
438	>	self.list_of_args = []
439	>	for i in range(self.total_number_of_jobs):
440	>	## Since there is no input, any site is good
441	>	self.jobDestination.append(["Any"])
442	>	if (self.sourceSeed):
443	>	if (self.sourceSeedVtx):
444	>	## pythia + vtx random seed
445	>	self.list_of_args.append([
446	>	str(self.sourceSeed)+str(i),
447	>	str(self.sourceSeedVtx)+str(i)
448	>	])
449	>	else:
450	>	## only pythia random seed
451	>	self.list_of_args.append([(str(self.sourceSeed)+str(i))])
452	>	else:
453	>	## no random seed
454	>	self.list_of_args.append([str(i)])
455	>	#print self.list_of_args
456	>
457		return
238	–
239	–	def checkBlackList(self, allSites):
240	–	if len(self.reCEBlackList)==0: return allSites
241	–	sites = []
242	–	for site in allSites:
243	–	common.logger.debug(10,'Site '+site)
244	–	good=1
245	–	for re in self.reCEBlackList:
246	–	if re.search(site):
247	–	common.logger.message('CE in black list, skipping site '+site)
248	–	good=0
249	–	pass
250	–	if good: sites.append(site)
251	–	if len(sites) == 0:
252	–	common.logger.debug(3,"No sites found after BlackList")
253	–	return sites
458
459	<	def checkWhiteList(self, allsites):
459	>	def split(self, jobParams):
460	>
461	>	common.jobDB.load()
462	>	#### Fabio
463	>	njobs = self.total_number_of_jobs
464	>	arglist = self.list_of_args
465	>	# create the empty structure
466	>	for i in range(njobs):
467	>	jobParams.append("")
468	>
469	>	for job in range(njobs):
470	>	jobParams[job] = arglist[job]
471	>	# print str(arglist[job])
472	>	# print jobParams[job]
473	>	common.jobDB.setArguments(job, jobParams[job])
474	>	common.logger.debug(5,"Job "+str(job)+" Destination: "+str(self.jobDestination[job]))
475	>	common.jobDB.setDestination(job, self.jobDestination[job])
476
477	<	if len(self.reCEWhiteList)==0: return pubDBUrls
478	<	sites = []
479	<	for site in allsites:
480	<	#print 'connecting to the URL ',url
481	<	good=0
482	<	for re in self.reCEWhiteList:
483	<	if re.search(site):
484	<	common.logger.debug(5,'CE in white list, adding site '+site)
485	<	good=1
486	<	if not good: continue
487	<	sites.append(site)
488	<	if len(sites) == 0:
269	<	common.logger.message("No sites found after WhiteList\n")
270	<	else:
271	<	common.logger.debug(5,"Selected sites via WhiteList are "+str(sites)+"\n")
272	<	return sites
477	>	common.jobDB.save()
478	>	return
479	>
480	>	def getJobTypeArguments(self, nj, sched):
481	>	result = ''
482	>	for i in common.jobDB.arguments(nj):
483	>	result=result+str(i)+" "
484	>	return result
485	>
486	>	def numberOfJobs(self):
487	>	# Fabio
488	>	return self.total_number_of_jobs
489
490		def getTarBall(self, exe):
491		"""
#	Line 327 | Line 543 | class Cmssw(JobType):
543		if os.path.exists(lib):
544		filesToBeTarred.append(libDir)
545
546	+	## Now check if module dir is present
547	+	moduleDir = 'module'
548	+	if os.path.isdir(swArea+'/'+moduleDir):
549	+	filesToBeTarred.append(moduleDir)
550	+
551		## Now check if the Data dir is present
552		dataDir = 'src/Data/'
553		if os.path.isdir(swArea+'/'+dataDir):
#	Line 354 | Line 575 | class Cmssw(JobType):
575		the execution environment for the job 'nj'.
576		"""
577		# Prepare JobType-independent part
578	<	txt = self.wsSetupCMSEnvironment_()
578	>	txt = ''
579	>
580	>	## OLI_Daniele at this level  middleware already known
581	>
582	>	txt += 'if [ $middleware == LCG ]; then \n'
583	>	txt += self.wsSetupCMSLCGEnvironment_()
584	>	txt += 'elif [ $middleware == OSG ]; then\n'
585	>	txt += '    time=`date -u +"%s"`\n'
586	>	txt += '    WORKING_DIR=$OSG_WN_TMP/cms_$time\n'
587	>	txt += '    echo "Creating working directory: $WORKING_DIR"\n'
588	>	txt += '    /bin/mkdir -p $WORKING_DIR\n'
589	>	txt += '    if [ ! -d $WORKING_DIR ] ;then\n'
590	>	txt += '        echo "SET_CMS_ENV 10016 ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
591	>	txt += '        echo "JOB_EXIT_STATUS = 10016"\n'
592	>	txt += '        echo "JobExitCode=10016" | tee -a $RUNTIME_AREA/$repo\n'
593	>	txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
594	>	txt += '        rm -f $RUNTIME_AREA/$repo \n'
595	>	txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
596	>	txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
597	>	txt += '        exit 1\n'
598	>	txt += '    fi\n'
599	>	txt += '\n'
600	>	txt += '    echo "Change to working directory: $WORKING_DIR"\n'
601	>	txt += '    cd $WORKING_DIR\n'
602	>	txt += self.wsSetupCMSOSGEnvironment_()
603	>	txt += 'fi\n'
604
605		# Prepare JobType-specific part
606		scram = self.scram.commandName()
#	Line 363 | Line 609 | class Cmssw(JobType):
609		txt += scram+' project CMSSW '+self.version+'\n'
610		txt += 'status=$?\n'
611		txt += 'if [ $status != 0 ] ; then\n'
612	<	txt += '   echo "SET_EXE_ENV 1 ==>ERROR CMSSW '+self.version+' not found on `hostname`" \n'
613	<	txt += '   echo "JOB_EXIT_STATUS = 5"\n'
614	<	txt += '   echo "SanityCheckCode = 5" | tee -a $RUNTIME_AREA/$repo\n'
612	>	txt += '   echo "SET_EXE_ENV 10034 ==>ERROR CMSSW '+self.version+' not found on `hostname`" \n'
613	>	txt += '   echo "JOB_EXIT_STATUS = 10034"\n'
614	>	txt += '   echo "JobExitCode=10034" | tee -a $RUNTIME_AREA/$repo\n'
615		txt += '   dumpStatus $RUNTIME_AREA/$repo\n'
616	<	txt += '   exit 5 \n'
616	>	txt += '   rm -f $RUNTIME_AREA/$repo \n'
617	>	txt += '   echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
618	>	txt += '   echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
619	>	## OLI_Daniele
620	>	txt += '    if [ $middleware == OSG ]; then \n'
621	>	txt += '        echo "Remove working directory: $WORKING_DIR"\n'
622	>	txt += '        cd $RUNTIME_AREA\n'
623	>	txt += '        /bin/rm -rf $WORKING_DIR\n'
624	>	txt += '        if [ -d $WORKING_DIR ] ;then\n'
625	>	txt += '            echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
626	>	txt += '            echo "JOB_EXIT_STATUS = 10018"\n'
627	>	txt += '            echo "JobExitCode=10018" | tee -a $RUNTIME_AREA/$repo\n'
628	>	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
629	>	txt += '            rm -f $RUNTIME_AREA/$repo \n'
630	>	txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
631	>	txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
632	>	txt += '        fi\n'
633	>	txt += '    fi \n'
634	>	txt += '   exit 1 \n'
635		txt += 'fi \n'
636		txt += 'echo "CMSSW_VERSION =  '+self.version+'"\n'
637		txt += 'cd '+self.version+'\n'
#	Line 376 | Line 640 | class Cmssw(JobType):
640
641		# Handle the arguments:
642		txt += "\n"
643	<	txt += "## ARGUMNETS: $1 Job Number\n"
380	<	# txt += "## ARGUMNETS: $2 First Event for this job\n"
381	<	# txt += "## ARGUMNETS: $3 Max Event for this job\n"
643	>	txt += "## number of arguments (first argument always jobnumber)\n"
644		txt += "\n"
645	<	txt += "narg=$#\n"
646	<	txt += "if [ $narg -lt 1 ]\n"
645	>	#        txt += "narg=$#\n"
646	>	txt += "if [ $nargs -lt 2 ]\n"
647		txt += "then\n"
648	<	txt += "    echo 'SET_EXE_ENV 1 ==> ERROR Too few arguments' +$narg+ \n"
649	<	txt += '    echo "JOB_EXIT_STATUS = 1"\n'
650	<	txt += '    echo "SanityCheckCode = 1" | tee -a $RUNTIME_AREA/$repo\n'
648	>	txt += "    echo 'SET_EXE_ENV 1 ==> ERROR Too few arguments' +$nargs+ \n"
649	>	txt += '    echo "JOB_EXIT_STATUS = 50113"\n'
650	>	txt += '    echo "JobExitCode=50113" | tee -a $RUNTIME_AREA/$repo\n'
651		txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
652	+	txt += '    rm -f $RUNTIME_AREA/$repo \n'
653	+	txt += '    echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
654	+	txt += '    echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
655	+	## OLI_Daniele
656	+	txt += '    if [ $middleware == OSG ]; then \n'
657	+	txt += '        echo "Remove working directory: $WORKING_DIR"\n'
658	+	txt += '        cd $RUNTIME_AREA\n'
659	+	txt += '        /bin/rm -rf $WORKING_DIR\n'
660	+	txt += '        if [ -d $WORKING_DIR ] ;then\n'
661	+	txt += '            echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
662	+	txt += '            echo "JOB_EXIT_STATUS = 50114"\n'
663	+	txt += '            echo "JobExitCode=50114" | tee -a $RUNTIME_AREA/$repo\n'
664	+	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
665	+	txt += '            rm -f $RUNTIME_AREA/$repo \n'
666	+	txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
667	+	txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
668	+	txt += '        fi\n'
669	+	txt += '    fi \n'
670		txt += "    exit 1\n"
671		txt += "fi\n"
672		txt += "\n"
393	–	txt += "NJob=$1\n"
394	–	# txt += "FirstEvent=$2\n"
395	–	# txt += "MaxEvents=$3\n"
673
674		# Prepare job-specific part
675		job = common.job_list[nj]
676		pset = os.path.basename(job.configFilename())
677		txt += '\n'
678	<	txt += 'cp $RUNTIME_AREA/'+pset+' pset.cfg\n'
679	<	# txt += 'if [ -e $RUNTIME_AREA/orcarc_$CE ] ; then\n'
680	<	# txt += '  cat $RUNTIME_AREA/orcarc_$CE .orcarc >> .orcarc_tmp\n'
681	<	# txt += '  mv .orcarc_tmp .orcarc\n'
682	<	# txt += 'fi\n'
683	<	# txt += 'if [ -e $RUNTIME_AREA/init_$CE.sh ] ; then\n'
684	<	# txt += '  cp $RUNTIME_AREA/init_$CE.sh init.sh\n'
685	<	# txt += 'fi\n'
678	>	if (self.datasetPath): # standard job
679	>	#txt += 'InputFiles=$2\n'
680	>	txt += 'InputFiles=${args[1]}\n'
681	>	txt += 'echo "Inputfiles:<$InputFiles>"\n'
682	>	txt += 'sed "s#{\'INPUT\'}#$InputFiles#" $RUNTIME_AREA/'+pset+' > pset.cfg\n'
683	>	else:  # pythia like job
684	>	if (self.sourceSeed):
685	>	#                txt += 'Seed=$2\n'
686	>	txt += 'Seed=${args[1]}\n'
687	>	txt += 'echo "Seed: <$Seed>"\n'
688	>	txt += 'sed "s#\<INPUT\>#$Seed#" $RUNTIME_AREA/'+pset+' > tmp.cfg\n'
689	>	if (self.sourceSeedVtx):
690	>	#                    txt += 'VtxSeed=$3\n'
691	>	txt += 'VtxSeed=${args[2]}\n'
692	>	txt += 'echo "VtxSeed: <$VtxSeed>"\n'
693	>	txt += 'sed "s#INPUTVTX#$VtxSeed#" tmp.cfg > pset.cfg\n'
694	>	else:
695	>	txt += 'mv tmp.cfg pset.cfg\n'
696	>	else:
697	>	txt += '# Copy untouched pset\n'
698	>	txt += 'cp $RUNTIME_AREA/'+pset+' pset.cfg\n'
699	>
700
701		if len(self.additional_inbox_files) > 0:
702		for file in self.additional_inbox_files:
703	<	txt += 'if [ -e $RUNTIME_AREA/'+file+' ] ; then\n'
704	<	txt += '   cp $RUNTIME_AREA/'+file+' .\n'
705	<	txt += '   chmod +x '+file+'\n'
703	>	relFile = file.split("/")[-1]
704	>	txt += 'if [ -e $RUNTIME_AREA/'+relFile+' ] ; then\n'
705	>	txt += '   cp $RUNTIME_AREA/'+relFile+' .\n'
706	>	txt += '   chmod +x '+relFile+'\n'
707		txt += 'fi\n'
708		pass
709
418	–	# txt += '\n'
419	–	# txt += 'chmod +x ./init.sh\n'
420	–	# txt += './init.sh\n'
421	–	# txt += 'exitStatus=$?\n'
422	–	# txt += 'if [ $exitStatus != 0 ] ; then\n'
423	–	# txt += '  echo "SET_EXE_ENV 1 ==> ERROR StageIn init script failed"\n'
424	–	# txt += '  echo "JOB_EXIT_STATUS = $exitStatus" \n'
425	–	# txt += '  echo "SanityCheckCode = $exitStatus" | tee -a $RUNTIME_AREA/$repo\n'
426	–	# txt += '  dumpStatus $RUNTIME_AREA/$repo\n'
427	–	# txt += '  exit $exitStatus\n'
428	–	# txt += 'fi\n'
429	–	# txt += "echo 'SET_EXE_ENV 0 ==> job setup ok'\n"
710		txt += 'echo "### END JOB SETUP ENVIRONMENT ###"\n\n'
711
432	–	# txt += 'echo "FirstEvent=$FirstEvent" >> .orcarc\n'
433	–	# txt += 'echo "MaxEvents=$MaxEvents" >> .orcarc\n'
434	–	# if self.ML:
435	–	#     txt += 'echo "MonalisaJobId=$NJob" >> .orcarc\n'
436	–
712		txt += '\n'
713		txt += 'echo "***** cat pset.cfg *********"\n'
714		txt += 'cat pset.cfg\n'
715		txt += 'echo "****** end pset.cfg ********"\n'
716	+	txt += '\n'
717	+	# txt += 'echo "***** cat pset1.cfg *********"\n'
718	+	# txt += 'cat pset1.cfg\n'
719	+	# txt += 'echo "****** end pset1.cfg ********"\n'
720	+	return txt
721	+
722	+	def wsBuildExe(self, nj):
723	+	"""
724	+	Put in the script the commands to build an executable
725	+	or a library.
726	+	"""
727	+
728	+	txt = ""
729	+
730	+	if os.path.isfile(self.tgzNameWithPath):
731	+	txt += 'echo "tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'"\n'
732	+	txt += 'tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
733	+	txt += 'untar_status=$? \n'
734	+	txt += 'if [ $untar_status -ne 0 ]; then \n'
735	+	txt += '   echo "SET_EXE 1 ==> ERROR Untarring .tgz file failed"\n'
736	+	txt += '   echo "JOB_EXIT_STATUS = $untar_status" \n'
737	+	txt += '   echo "JobExitCode=$untar_status" | tee -a $RUNTIME_AREA/$repo\n'
738	+	txt += '   if [ $middleware == OSG ]; then \n'
739	+	txt += '       echo "Remove working directory: $WORKING_DIR"\n'
740	+	txt += '       cd $RUNTIME_AREA\n'
741	+	txt += '       /bin/rm -rf $WORKING_DIR\n'
742	+	txt += '       if [ -d $WORKING_DIR ] ;then\n'
743	+	txt += '           echo "SET_EXE 50999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Untarring .tgz file failed"\n'
744	+	txt += '           echo "JOB_EXIT_STATUS = 50999"\n'
745	+	txt += '           echo "JobExitCode=50999" | tee -a $RUNTIME_AREA/$repo\n'
746	+	txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
747	+	txt += '           rm -f $RUNTIME_AREA/$repo \n'
748	+	txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
749	+	txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
750	+	txt += '       fi\n'
751	+	txt += '   fi \n'
752	+	txt += '   \n'
753	+	txt += '   exit 1 \n'
754	+	txt += 'else \n'
755	+	txt += '   echo "Successful untar" \n'
756	+	txt += 'fi \n'
757	+	pass
758	+
759		return txt
760
761		def modifySteeringCards(self, nj):
#	Line 450 | Line 768 | class Cmssw(JobType):
768		return self.executable
769
770		def executableArgs(self):
771	<	return "-p pset.cfg"
771	>	return " -p pset.cfg"
772
773		def inputSandbox(self, nj):
774		"""
#	Line 465 | Line 783 | class Cmssw(JobType):
783		## config
784		inp_box.append(common.job_list[nj].configFilename())
785		## additional input files
786	<	for file in self.additional_inbox_files:
787	<	inp_box.append(common.work_space.cwdDir()+file)
470	<	#print "sono inputSandbox, inp_box = ", inp_box
786	>	#for file in self.additional_inbox_files:
787	>	#    inp_box.append(common.work_space.cwdDir()+file)
788		return inp_box
789
790		def outputSandbox(self, nj):
#	Line 490 | Line 807 | class Cmssw(JobType):
807		"""
808		Make initial modifications of the user's steering card file.
809		"""
493	–	infile = open(self.pset,'r')
494	–
495	–	outfile = open(common.work_space.jobDir()+self.name()+'.cfg', 'w')
496	–
497	–	outfile.write('\n\n##### The following cards have been created by CRAB: DO NOT TOUCH #####\n')
498	–
499	–	outfile.write('InputCollections=/System/'+self.owner+'/'+self.dataset+'/'+self.dataset+'\n')
500	–
501	–	infile.close()
502	–	outfile.close()
810		return
811
812		def wsRenameOutput(self, nj):
#	Line 508 | Line 815 | class Cmssw(JobType):
815		"""
816
817		txt = '\n'
818	+	txt += '# directory content\n'
819	+	txt += 'ls \n'
820		file_list = ''
821		for fileWithSuffix in self.output_file:
822		output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
823		file_list=file_list+output_file_num+' '
824		txt += '\n'
825	<	txt += 'ls \n'
517	<	txt += '\n'
825	>	txt += '# check output file\n'
826		txt += 'ls '+fileWithSuffix+'\n'
827	<	txt += 'exe_result=$?\n'
828	<	txt += 'if [ $exe_result -ne 0 ] ; then\n'
829	<	txt += '   echo "ERROR: No output file to manage"\n'
830	<	txt += '   echo "JOB_EXIT_STATUS = $exe_result"\n'
831	<	txt += '   echo "SanityCheckCode = $exe_result" | tee -a $RUNTIME_AREA/$repo\n'
832	<	txt += '   dumpStatus $RUNTIME_AREA/$repo\n'
833	<	txt += '   exit $exe_result \n'
827	>	txt += 'ls_result=$?\n'
828	>	#txt += 'exe_result=$?\n'
829	>	txt += 'if [ $ls_result -ne 0 ] ; then\n'
830	>	txt += '   echo "ERROR: Problem with output file"\n'
831	>	#txt += '   echo "JOB_EXIT_STATUS = $exe_result"\n'
832	>	#txt += '   echo "JobExitCode=60302" | tee -a $RUNTIME_AREA/$repo\n'
833	>	#txt += '   dumpStatus $RUNTIME_AREA/$repo\n'
834	>	### OLI_DANIELE
835	>	if common.scheduler.boss_scheduler_name == 'condor_g':
836	>	txt += '    if [ $middleware == OSG ]; then \n'
837	>	txt += '        echo "prepare dummy output file"\n'
838	>	txt += '        echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
839	>	txt += '    fi \n'
840		txt += 'else\n'
841		txt += '   cp '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
842		txt += 'fi\n'
529	–	txt += 'cd $RUNTIME_AREA\n'
530	–
531	–	pass
843
844	+	txt += 'cd $RUNTIME_AREA\n'
845		file_list=file_list[:-1]
846		txt += 'file_list="'+file_list+'"\n'
847	+	txt += 'cd $RUNTIME_AREA\n'
848	+	### OLI_DANIELE
849	+	txt += 'if [ $middleware == OSG ]; then\n'
850	+	txt += '    cd $RUNTIME_AREA\n'
851	+	txt += '    echo "Remove working directory: $WORKING_DIR"\n'
852	+	txt += '    /bin/rm -rf $WORKING_DIR\n'
853	+	txt += '    if [ -d $WORKING_DIR ] ;then\n'
854	+	txt += '        echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
855	+	txt += '        echo "JOB_EXIT_STATUS = 60999"\n'
856	+	txt += '        echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
857	+	txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
858	+	txt += '        rm -f $RUNTIME_AREA/$repo \n'
859	+	txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
860	+	txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
861	+	txt += '    fi\n'
862	+	txt += 'fi\n'
863	+	txt += '\n'
864		return txt
865
866		def numberFile_(self, file, txt):
#	Line 554 | Line 883 | class Cmssw(JobType):
883
884		return result
885
886	<	def getRequirements(self):
886	>	def getRequirements(self, nj):
887		"""
888		return job requirements to add to jdl files
889		"""
890		req = ''
891	<	if common.analisys_common_info['sites']:
892	<	if common.analisys_common_info['sw_version']:
893	<	req='Member("VO-cms-' + \
894	<	common.analisys_common_info['sw_version'] + \
895	<	'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
896	<	if len(common.analisys_common_info['sites'])>0:
897	<	req = req + ' && ('
898	<	for i in range(len(common.analisys_common_info['sites'])):
899	<	req = req + 'other.GlueCEInfoHostName == "' \
900	<	+ common.analisys_common_info['sites'][i] + '"'
901	<	if ( i < (int(len(common.analisys_common_info['sites']) - 1)) ):
902	<	req = req + ' || '
903	<	req = req + ')'
904	<	#print "req = ", req
891	>	if common.analisys_common_info['sw_version']:
892	>	req='Member("VO-cms-' + \
893	>	common.analisys_common_info['sw_version'] + \
894	>	'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
895	>
896	>	req = req + ' && (other.GlueHostNetworkAdapterOutboundIP)'
897	>
898	>	## here we should get the requirement for job nj
899	>	sites = common.jobDB.destination(nj)
900	>
901	>	# check for "Any" site, in case no requirement for site
902	>	if len(sites)>0 and sites[0]!="Any":
903	>	req = req + ' && anyMatch(other.storage.CloseSEs, ('
904	>	for site in sites:
905	>	#req = req + 'other.GlueCEInfoHostName == "' + site + '" || '
906	>	req = req + 'target.GlueSEUniqueID=="' + site + '" || '
907	>	pass
908	>	# remove last ||
909	>	req = req[0:-4]
910	>	req = req + '))'
911	>
912		return req
913	+
914	+	def configFilename(self):
915	+	""" return the config filename """
916	+	return self.name()+'.cfg'
917	+
918	+	### OLI_DANIELE
919	+	def wsSetupCMSOSGEnvironment_(self):
920	+	"""
921	+	Returns part of a job script which is prepares
922	+	the execution environment and which is common for all CMS jobs.
923	+	"""
924	+	txt = '\n'
925	+	txt += '   echo "### SETUP CMS OSG  ENVIRONMENT ###"\n'
926	+	txt += '   if [ -f $GRID3_APP_DIR/cmssoft/cmsset_default.sh ] ;then\n'
927	+	txt += '      # Use $GRID3_APP_DIR/cmssoft/cmsset_default.sh to setup cms software\n'
928	+	txt += '       source $GRID3_APP_DIR/cmssoft/cmsset_default.sh '+self.version+'\n'
929	+	txt += '   elif [ -f $OSG_APP/cmssoft/cmsset_default.sh ] ;then\n'
930	+	txt += '      # Use $OSG_APP/cmssoft/cmsset_default.sh to setup cms software\n'
931	+	txt += '       source $OSG_APP/cmssoft/cmsset_default.sh '+self.version+'\n'
932	+	txt += '   else\n'
933	+	txt += '       echo "SET_CMS_ENV 10020 ==> ERROR $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cmsset_default.sh file not found"\n'
934	+	txt += '       echo "JOB_EXIT_STATUS = 10020"\n'
935	+	txt += '       echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
936	+	txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
937	+	txt += '       rm -f $RUNTIME_AREA/$repo \n'
938	+	txt += '       echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
939	+	txt += '       echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
940	+	txt += '       exit 1\n'
941	+	txt += '\n'
942	+	txt += '       echo "Remove working directory: $WORKING_DIR"\n'
943	+	txt += '       cd $RUNTIME_AREA\n'
944	+	txt += '       /bin/rm -rf $WORKING_DIR\n'
945	+	txt += '       if [ -d $WORKING_DIR ] ;then\n'
946	+	txt += '            echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cmsset_default.sh file not found"\n'
947	+	txt += '            echo "JOB_EXIT_STATUS = 10017"\n'
948	+	txt += '            echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n'
949	+	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
950	+	txt += '            rm -f $RUNTIME_AREA/$repo \n'
951	+	txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
952	+	txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
953	+	txt += '       fi\n'
954	+	txt += '\n'
955	+	txt += '       exit 1\n'
956	+	txt += '   fi\n'
957	+	txt += '\n'
958	+	txt += '   echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
959	+	txt += '   echo " END SETUP CMS OSG  ENVIRONMENT "\n'
960	+
961	+	return txt
962	+
963	+	### OLI_DANIELE
964	+	def wsSetupCMSLCGEnvironment_(self):
965	+	"""
966	+	Returns part of a job script which is prepares
967	+	the execution environment and which is common for all CMS jobs.
968	+	"""
969	+	txt  = '   \n'
970	+	txt += '   echo " ### SETUP CMS LCG  ENVIRONMENT ### "\n'
971	+	txt += '   if [ ! $VO_CMS_SW_DIR ] ;then\n'
972	+	txt += '       echo "SET_CMS_ENV 10031 ==> ERROR CMS software dir not found on WN `hostname`"\n'
973	+	txt += '       echo "JOB_EXIT_STATUS = 10031" \n'
974	+	txt += '       echo "JobExitCode=10031" | tee -a $RUNTIME_AREA/$repo\n'
975	+	txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
976	+	txt += '       rm -f $RUNTIME_AREA/$repo \n'
977	+	txt += '       echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
978	+	txt += '       echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
979	+	txt += '       exit 1\n'
980	+	txt += '   else\n'
981	+	txt += '       echo "Sourcing environment... "\n'
982	+	txt += '       if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
983	+	txt += '           echo "SET_CMS_ENV 10020 ==> ERROR cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
984	+	txt += '           echo "JOB_EXIT_STATUS = 10020"\n'
985	+	txt += '           echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
986	+	txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
987	+	txt += '           rm -f $RUNTIME_AREA/$repo \n'
988	+	txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
989	+	txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
990	+	txt += '           exit 1\n'
991	+	txt += '       fi\n'
992	+	txt += '       echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
993	+	txt += '       source $VO_CMS_SW_DIR/cmsset_default.sh\n'
994	+	txt += '       result=$?\n'
995	+	txt += '       if [ $result -ne 0 ]; then\n'
996	+	txt += '           echo "SET_CMS_ENV 10032 ==> ERROR problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
997	+	txt += '           echo "JOB_EXIT_STATUS = 10032"\n'
998	+	txt += '           echo "JobExitCode=10032" | tee -a $RUNTIME_AREA/$repo\n'
999	+	txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1000	+	txt += '           rm -f $RUNTIME_AREA/$repo \n'
1001	+	txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1002	+	txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1003	+	txt += '           exit 1\n'
1004	+	txt += '       fi\n'
1005	+	txt += '   fi\n'
1006	+	txt += '   \n'
1007	+	txt += '   string=`cat /etc/redhat-release`\n'
1008	+	txt += '   echo $string\n'
1009	+	txt += '   if [[ $string = *alhalla* ]]; then\n'
1010	+	txt += '       echo "SCRAM_ARCH= $SCRAM_ARCH"\n'
1011	+	txt += '   elif [[ $string = *Enterprise* ]] || [[ $string = *cientific* ]]; then\n'
1012	+	txt += '       export SCRAM_ARCH=slc3_ia32_gcc323\n'
1013	+	txt += '       echo "SCRAM_ARCH= $SCRAM_ARCH"\n'
1014	+	txt += '   else\n'
1015	+	txt += '       echo "SET_CMS_ENV 10033 ==> ERROR OS unknown, LCG environment not initialized"\n'
1016	+	txt += '       echo "JOB_EXIT_STATUS = 10033"\n'
1017	+	txt += '       echo "JobExitCode=10033" | tee -a $RUNTIME_AREA/$repo\n'
1018	+	txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
1019	+	txt += '       rm -f $RUNTIME_AREA/$repo \n'
1020	+	txt += '       echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1021	+	txt += '       echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1022	+	txt += '       exit 1\n'
1023	+	txt += '   fi\n'
1024	+	txt += '   echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1025	+	txt += '   echo "### END SETUP CMS LCG ENVIRONMENT ###"\n'
1026	+	return txt
1027	+
1028	+	def setParam_(self, param, value):
1029	+	self._params[param] = value
1030	+
1031	+	def getParams(self):
1032	+	return self._params
1033	+
1034	+	def setTaskid_(self):
1035	+	self._taskId = self.cfg_params['taskId']
1036	+
1037	+	def getTaskid(self):
1038	+	return self._taskId
1039	+
1040	+	#######################################################################
1041	+	def uniquelist(self, old):
1042	+	"""
1043	+	remove duplicates from a list
1044	+	"""
1045	+	nd={}
1046	+	for e in old:
1047	+	nd[e]=0
1048	+	return nd.keys()

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines