ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/cms_cmssw.py

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.73 by gutsche, Sun Apr 8 18:39:51 2007 UTC vs.
Revision 1.185 by ewv, Wed May 7 14:56:24 2008 UTC

#	Line 2 | Line 2 | from JobType import JobType
2		from crab_logger import Logger
3		from crab_exceptions import *
4		from crab_util import *
5	+	from BlackWhiteListParser import BlackWhiteListParser
6		import common
6	–	import PsetManipulator
7	–	import DataDiscovery
8	–	import DataDiscovery_DBS2
9	–	import DataLocation
7		import Scram
8	+	from LFNBaseName import *
9
10	<	import os, string, re, shutil, glob
10	>	import os, string, glob
11
12		class Cmssw(JobType):
13		def __init__(self, cfg_params, ncjobs):
14		JobType.__init__(self, 'CMSSW')
15		common.logger.debug(3,'CMSSW::__init__')
16
17	<	# Marco.
17	>	self.argsList = []
18	>
19		self._params = {}
20		self.cfg_params = cfg_params
21	+	# init BlackWhiteListParser
22	+	self.blackWhiteListParser = BlackWhiteListParser(cfg_params)
23
24	<	try:
24	<	self.MaxTarBallSize = float(self.cfg_params['EDG.maxtarballsize'])
25	<	except KeyError:
26	<	self.MaxTarBallSize = 100.0
24	>	self.MaxTarBallSize = float(self.cfg_params.get('EDG.maxtarballsize',9.5))
25
26		# number of jobs requested to be created, limit obj splitting
27		self.ncjobs = ncjobs
28
29		log = common.logger
30	<
30	>
31		self.scram = Scram.Scram(cfg_params)
32		self.additional_inbox_files = []
33		self.scriptExe = ''
#	Line 37 | Line 35 | class Cmssw(JobType):
35		self.executable_arch = self.scram.getArch()
36		self.tgz_name = 'default.tgz'
37		self.scriptName = 'CMSSW.sh'
38	<	self.pset = ''      #scrip use case Da
38	>	self.pset = ''      #scrip use case Da
39		self.datasetPath = '' #scrip use case Da
40
41		# set FJR file name
42		self.fjrFileName = 'crab_fjr.xml'
43
44		self.version = self.scram.getSWVersion()
45	<	common.taskDB.setDict('codeVersion',self.version)
46	<	self.setParam_('application', self.version)
45	>	version_array = self.version.split('_')
46	>	self.CMSSW_major = 0
47	>	self.CMSSW_minor = 0
48	>	self.CMSSW_patch = 0
49	>	try:
50	>	self.CMSSW_major = int(version_array[1])
51	>	self.CMSSW_minor = int(version_array[2])
52	>	self.CMSSW_patch = int(version_array[3])
53	>	except:
54	>	msg = "Cannot parse CMSSW version string: " + self.version + " for major and minor release number!"
55	>	raise CrabException(msg)
56
57		### collect Data cards
58
59	<	## get DBS mode
60	<	try:
54	<	self.use_dbs_2 = int(self.cfg_params['CMSSW.use_dbs_2'])
55	<	except KeyError:
56	<	self.use_dbs_2 = 0
57	<
58	<	try:
59	<	tmp =  cfg_params['CMSSW.datasetpath']
60	<	log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
61	<	if string.lower(tmp)=='none':
62	<	self.datasetPath = None
63	<	self.selectNoInput = 1
64	<	else:
65	<	self.datasetPath = tmp
66	<	self.selectNoInput = 0
67	<	except KeyError:
68	<	msg = "Error: datasetpath not defined "
59	>	if not cfg_params.has_key('CMSSW.datasetpath'):
60	>	msg = "Error: datasetpath not defined "
61		raise CrabException(msg)
62	<
63	<	# ML monitoring
64	<	# split dataset path style: /PreProdR3Minbias/SIM/GEN-SIM
65	<	if not self.datasetPath:
66	<	self.setParam_('dataset', 'None')
75	<	self.setParam_('owner', 'None')
62	>	tmp =  cfg_params['CMSSW.datasetpath']
63	>	log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
64	>	if string.lower(tmp)=='none':
65	>	self.datasetPath = None
66	>	self.selectNoInput = 1
67		else:
68	<	datasetpath_split = self.datasetPath.split("/")
69	<	self.setParam_('dataset', datasetpath_split[1])
79	<	self.setParam_('owner', datasetpath_split[-1])
80	<
81	<	self.setTaskid_()
82	<	self.setParam_('taskId', self.cfg_params['taskId'])
68	>	self.datasetPath = tmp
69	>	self.selectNoInput = 0
70
71		self.dataTiers = []
72
73		## now the application
74	<	try:
75	<	self.executable = cfg_params['CMSSW.executable']
89	<	self.setParam_('exe', self.executable)
90	<	log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable)
91	<	msg = "Default executable cmsRun overridden. Switch to " + self.executable
92	<	log.debug(3,msg)
93	<	except KeyError:
94	<	self.executable = 'cmsRun'
95	<	self.setParam_('exe', self.executable)
96	<	msg = "User executable not defined. Use cmsRun"
97	<	log.debug(3,msg)
98	<	pass
74	>	self.executable = cfg_params.get('CMSSW.executable','cmsRun')
75	>	log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable)
76
77	<	try:
101	<	self.pset = cfg_params['CMSSW.pset']
102	<	log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset)
103	<	if self.pset.lower() != 'none' :
104	<	if (not os.path.exists(self.pset)):
105	<	raise CrabException("User defined PSet file "+self.pset+" does not exist")
106	<	else:
107	<	self.pset = None
108	<	except KeyError:
77	>	if not cfg_params.has_key('CMSSW.pset'):
78		raise CrabException("PSet file missing. Cannot run cmsRun ")
79	+	self.pset = cfg_params['CMSSW.pset']
80	+	log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset)
81	+	if self.pset.lower() != 'none' :
82	+	if (not os.path.exists(self.pset)):
83	+	raise CrabException("User defined PSet file "+self.pset+" does not exist")
84	+	else:
85	+	self.pset = None
86
87		# output files
88		## stuff which must be returned always via sandbox
#	Line 116 | Line 92 | class Cmssw(JobType):
92		self.output_file_sandbox.append(self.fjrFileName)
93
94		# other output files to be returned via sandbox or copied to SE
95	<	try:
96	<	self.output_file = []
97	<	tmp = cfg_params['CMSSW.output_file']
98	<	if tmp != '':
99	<	tmpOutFiles = string.split(cfg_params['CMSSW.output_file'],',')
100	<	log.debug(7, 'cmssw::cmssw(): output files '+str(tmpOutFiles))
101	<	for tmp in tmpOutFiles:
102	<	tmp=string.strip(tmp)
127	<	self.output_file.append(tmp)
128	<	pass
129	<	else:
130	<	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available")
95	>	self.output_file = []
96	>	tmp = cfg_params.get('CMSSW.output_file',None)
97	>	if tmp :
98	>	tmpOutFiles = string.split(tmp,',')
99	>	log.debug(7, 'cmssw::cmssw(): output files '+str(tmpOutFiles))
100	>	for tmp in tmpOutFiles:
101	>	tmp=string.strip(tmp)
102	>	self.output_file.append(tmp)
103		pass
104	<	pass
105	<	except KeyError:
106	<	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available")
135	<	pass
104	>	else:
105	>	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
106	>	pass
107
108		# script_exe file as additional file in inputSandbox
109	<	try:
110	<	self.scriptExe = cfg_params['USER.script_exe']
111	<	if self.scriptExe != '':
112	<	if not os.path.isfile(self.scriptExe):
113	<	msg ="ERROR. file "+self.scriptExe+" not found"
114	<	raise CrabException(msg)
144	<	self.additional_inbox_files.append(string.strip(self.scriptExe))
145	<	except KeyError:
146	<	self.scriptExe = ''
109	>	self.scriptExe = cfg_params.get('USER.script_exe',None)
110	>	if self.scriptExe :
111	>	if not os.path.isfile(self.scriptExe):
112	>	msg ="ERROR. file "+self.scriptExe+" not found"
113	>	raise CrabException(msg)
114	>	self.additional_inbox_files.append(string.strip(self.scriptExe))
115
116		#CarlosDaniele
117		if self.datasetPath == None and self.pset == None and self.scriptExe == '' :
118	<	msg ="Error. script_exe  not defined"
119	<	raise CrabException(msg)
118	>	msg ="Error. script_exe  not defined"
119	>	raise CrabException(msg)
120
121		## additional input files
122	<	try:
122	>	if cfg_params.has_key('USER.additional_input_files'):
123		tmpAddFiles = string.split(cfg_params['USER.additional_input_files'],',')
124		for tmp in tmpAddFiles:
125		tmp = string.strip(tmp)
126		dirname = ''
127		if not tmp[0]=="/": dirname = "."
128	<	files = glob.glob(os.path.join(dirname, tmp))
128	>	files = []
129	>	if string.find(tmp,"*")>-1:
130	>	files = glob.glob(os.path.join(dirname, tmp))
131	>	if len(files)==0:
132	>	raise CrabException("No additional input file found with this pattern: "+tmp)
133	>	else:
134	>	files.append(tmp)
135		for file in files:
136		if not os.path.exists(file):
137		raise CrabException("Additional input file not found: "+file)
138		pass
139	<	storedFile = common.work_space.shareDir()+file
140	<	shutil.copyfile(file, storedFile)
141	<	self.additional_inbox_files.append(string.strip(storedFile))
139	>	# fname = string.split(file, '/')[-1]
140	>	# storedFile = common.work_space.pathForTgz()+'share/'+fname
141	>	# shutil.copyfile(file, storedFile)
142	>	self.additional_inbox_files.append(string.strip(file))
143		pass
144		pass
145		common.logger.debug(5,"Additional input files: "+str(self.additional_inbox_files))
146	<	except KeyError:
172	<	pass
173	<
174	<	# files per job
175	<	try:
176	<	if (cfg_params['CMSSW.files_per_jobs']):
177	<	raise CrabException("files_per_jobs no longer supported.  Quitting.")
178	<	except KeyError:
179	<	pass
146	>	pass
147
148		## Events per job
149	<	try:
149	>	if cfg_params.has_key('CMSSW.events_per_job'):
150		self.eventsPerJob =int( cfg_params['CMSSW.events_per_job'])
151		self.selectEventsPerJob = 1
152	<	except KeyError:
152	>	else:
153		self.eventsPerJob = -1
154		self.selectEventsPerJob = 0
155	<
155	>
156		## number of jobs
157	<	try:
157	>	if cfg_params.has_key('CMSSW.number_of_jobs'):
158		self.theNumberOfJobs =int( cfg_params['CMSSW.number_of_jobs'])
159		self.selectNumberOfJobs = 1
160	<	except KeyError:
160	>	else:
161		self.theNumberOfJobs = 0
162		self.selectNumberOfJobs = 0
163
164	<	try:
164	>	if cfg_params.has_key('CMSSW.total_number_of_events'):
165		self.total_number_of_events = int(cfg_params['CMSSW.total_number_of_events'])
166		self.selectTotalNumberEvents = 1
167	<	except KeyError:
167	>	else:
168		self.total_number_of_events = 0
169		self.selectTotalNumberEvents = 0
170
171	<	if self.pset != None: #CarlosDaniele
171	>	if self.pset != None: #CarlosDaniele
172		if ( (self.selectTotalNumberEvents + self.selectEventsPerJob + self.selectNumberOfJobs) != 2 ):
173		msg = 'Must define exactly two of total_number_of_events, events_per_job, or number_of_jobs.'
174		raise CrabException(msg)
#	Line 210 | Line 177 | class Cmssw(JobType):
177		msg = 'Must specify  number_of_jobs.'
178		raise CrabException(msg)
179
180	<	## source seed for pythia
181	<	try:
182	<	self.sourceSeed = int(cfg_params['CMSSW.pythia_seed'])
183	<	except KeyError:
184	<	self.sourceSeed = None
185	<	common.logger.debug(5,"No seed given")
180	>	## New method of dealing with seeds
181	>	self.incrementSeeds = []
182	>	self.preserveSeeds = []
183	>	if cfg_params.has_key('CMSSW.preserve_seeds'):
184	>	tmpList = cfg_params['CMSSW.preserve_seeds'].split(',')
185	>	for tmp in tmpList:
186	>	tmp.strip()
187	>	self.preserveSeeds.append(tmp)
188	>	if cfg_params.has_key('CMSSW.increment_seeds'):
189	>	tmpList = cfg_params['CMSSW.increment_seeds'].split(',')
190	>	for tmp in tmpList:
191	>	tmp.strip()
192	>	self.incrementSeeds.append(tmp)
193	>
194	>	## Old method of dealing with seeds
195	>	## FUTURE: This is for old CMSSW and old CRAB. Can throw exceptions after a couple of CRAB releases and then
196	>	## remove
197	>	self.sourceSeed = cfg_params.get('CMSSW.pythia_seed',None)
198	>	if self.sourceSeed:
199	>	print "pythia_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
200	>	self.incrementSeeds.append('sourceSeed')
201	>	self.incrementSeeds.append('theSource')
202	>
203	>	self.sourceSeedVtx = cfg_params.get('CMSSW.vtx_seed',None)
204	>	if self.sourceSeedVtx:
205	>	print "vtx_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
206	>	self.incrementSeeds.append('VtxSmeared')
207	>
208	>	self.sourceSeedG4 = cfg_params.get('CMSSW.g4_seed',None)
209	>	if self.sourceSeedG4:
210	>	print "g4_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
211	>	self.incrementSeeds.append('g4SimHits')
212	>
213	>	self.sourceSeedMix = cfg_params.get('CMSSW.mix_seed',None)
214	>	if self.sourceSeedMix:
215	>	print "mix_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
216	>	self.incrementSeeds.append('mix')
217	>
218	>	self.firstRun = cfg_params.get('CMSSW.first_run',None)
219
220	–	try:
221	–	self.sourceSeedVtx = int(cfg_params['CMSSW.vtx_seed'])
222	–	except KeyError:
223	–	self.sourceSeedVtx = None
224	–	common.logger.debug(5,"No vertex seed given")
225	–	try:
226	–	self.firstRun = int(cfg_params['CMSSW.first_run'])
227	–	except KeyError:
228	–	self.firstRun = None
229	–	common.logger.debug(5,"No first run given")
220		if self.pset != None: #CarlosDaniele
221	<	self.PsetEdit = PsetManipulator.PsetManipulator(self.pset) #Daniele Pset
221	>	import PsetManipulator as pp
222	>	PsetEdit = pp.PsetManipulator(self.pset) #Daniele Pset
223	>
224	>	# Copy/return
225	>
226	>	self.copy_data = int(cfg_params.get('USER.copy_data',0))
227	>	self.return_data = int(cfg_params.get('USER.return_data',0))
228
229		#DBSDLS-start
230	<	## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
230	>	## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
231		self.maxEvents=0  # max events available   ( --> check the requested nb. of evts in Creator.py)
232		self.DBSPaths={}  # all dbs paths requested ( --> input to the site local discovery script)
233		self.jobDestination=[]  # Site destination(s) for each job (list of lists)
#	Line 240 | Line 236 | class Cmssw(JobType):
236		blockSites = {}
237		if self.datasetPath:
238		blockSites = self.DataDiscoveryAndLocation(cfg_params)
239	<	#DBSDLS-end
239	>	#DBSDLS-end
240	>
241
245	–	self.tgzNameWithPath = self.getTarBall(self.executable)
246	–
242		## Select Splitting
243	<	if self.selectNoInput:
243	>	if self.selectNoInput:
244		if self.pset == None: #CarlosDaniele
245		self.jobSplittingForScript()
246		else:
247		self.jobSplittingNoInput()
248	<	else:
248	>	else:
249		self.jobSplittingByBlocks(blockSites)
250
251		# modify Pset
252		if self.pset != None: #CarlosDaniele
253		try:
254	<	if (self.datasetPath): # standard job
255	<	# allow to processa a fraction of events in a file
256	<	self.PsetEdit.inputModule("INPUT")
257	<	self.PsetEdit.maxEvent("INPUTMAXEVENTS")
258	<	self.PsetEdit.skipEvent("INPUTSKIPEVENTS")
264	<	else:  # pythia like job
265	<	self.PsetEdit.maxEvent(self.eventsPerJob)
266	<	if (self.firstRun):
267	<	self.PsetEdit.pythiaFirstRun("INPUTFIRSTRUN")  #First Run
268	<	if (self.sourceSeed) :
269	<	self.PsetEdit.pythiaSeed("INPUT")
270	<	if (self.sourceSeedVtx) :
271	<	self.PsetEdit.pythiaSeedVtx("INPUTVTX")
272	<	# add FrameworkJobReport to parameter-set
273	<	self.PsetEdit.addCrabFJR(self.fjrFileName)
274	<	self.PsetEdit.psetWriter(self.configFilename())
254	>	# Add FrameworkJobReport to parameter-set, set max events.
255	>	# Reset later for data jobs by writeCFG which does all modifications
256	>	PsetEdit.addCrabFJR(self.fjrFileName) # FUTURE: Job report addition not needed by CMSSW>1.5
257	>	PsetEdit.maxEvent(self.eventsPerJob)
258	>	PsetEdit.psetWriter(self.configFilename())
259		except:
260	<	msg='Error while manipuliating ParameterSet: exiting...'
260	>	msg='Error while manipulating ParameterSet: exiting...'
261		raise CrabException(msg)
262	+	self.tgzNameWithPath = self.getTarBall(self.executable)
263
264		def DataDiscoveryAndLocation(self, cfg_params):
265
266	+	import DataDiscovery
267	+	import DataLocation
268		common.logger.debug(10,"CMSSW::DataDiscoveryAndLocation()")
269
270		datasetPath=self.datasetPath
271
272		## Contact the DBS
273	<	common.logger.message("Contacting DBS...")
273	>	common.logger.message("Contacting Data Discovery Services ...")
274		try:
275	<
289	<	if self.use_dbs_2 == 1 :
290	<	self.pubdata=DataDiscovery_DBS2.DataDiscovery_DBS2(datasetPath, cfg_params)
291	<	else :
292	<	self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params)
275	>	self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params)
276		self.pubdata.fetchDBSInfo()
277
278		except DataDiscovery.NotExistingDatasetError, ex :
#	Line 301 | Line 284 | class Cmssw(JobType):
284		except DataDiscovery.DataDiscoveryError, ex:
285		msg = 'ERROR ***: failed Data Discovery in DBS :  %s'%ex.getErrorMessage()
286		raise CrabException(msg)
304	–	except DataDiscovery_DBS2.NotExistingDatasetError_DBS2, ex :
305	–	msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
306	–	raise CrabException(msg)
307	–	except DataDiscovery_DBS2.NoDataTierinProvenanceError_DBS2, ex :
308	–	msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
309	–	raise CrabException(msg)
310	–	except DataDiscovery_DBS2.DataDiscoveryError_DBS2, ex:
311	–	msg = 'ERROR ***: failed Data Discovery in DBS :  %s'%ex.getErrorMessage()
312	–	raise CrabException(msg)
313	–
314	–	## get list of all required data in the form of dbs paths  (dbs path = /dataset/datatier/owner)
315	–	common.logger.message("Required data are :"+self.datasetPath)
287
288		self.filesbyblock=self.pubdata.getFiles()
289		self.eventsbyblock=self.pubdata.getEventsPerBlock()
290		self.eventsbyfile=self.pubdata.getEventsPerFile()
291
292		## get max number of events
293	<	self.maxEvents=self.pubdata.getMaxEvents() ##  self.maxEvents used in Creator.py
323	<	common.logger.message("The number of available events is %s\n"%self.maxEvents)
293	>	self.maxEvents=self.pubdata.getMaxEvents() ##  self.maxEvents used in Creator.py
294
325	–	common.logger.message("Contacting DLS...")
295		## Contact the DLS and build a list of sites hosting the fileblocks
296		try:
297		dataloc=DataLocation.DataLocation(self.filesbyblock.keys(),cfg_params)
#	Line 330 | Line 299 | class Cmssw(JobType):
299		except DataLocation.DataLocationError , ex:
300		msg = 'ERROR ***: failed Data Location in DLS \n %s '%ex.getErrorMessage()
301		raise CrabException(msg)
302	<
302	>
303
304		sites = dataloc.getSites()
305		allSites = []
#	Line 340 | Line 309 | class Cmssw(JobType):
309		allSites.append(oneSite)
310		allSites = self.uniquelist(allSites)
311
312	<	common.logger.message("Sites ("+str(len(allSites))+") hosting part/all of dataset: "+str(allSites))
313	<	common.logger.debug(6, "List of Sites: "+str(allSites))
312	>	# screen output
313	>	common.logger.message("Requested dataset: " + datasetPath + " has " + str(self.maxEvents) + " events in " + str(len(self.filesbyblock.keys())) + " blocks.\n")
314	>
315		return sites
316	<
316	>
317	>	# to Be Removed  DS -- BL
318	>	#  def setArgsList(self, argsList):
319	>	#      self.argsList = argsList
320	>
321		def jobSplittingByBlocks(self, blockSites):
322		"""
323		Perform job splitting. Jobs run over an integer number of files
#	Line 393 | Line 367 | class Cmssw(JobType):
367		totalNumberOfJobs = 999999999
368		else :
369		totalNumberOfJobs = self.ncjobs
396	–
370
371		blocks = blockSites.keys()
372		blockCount = 0
#	Line 403 | Line 376 | class Cmssw(JobType):
376		jobCount = 0
377		list_of_lists = []
378
379	+	# list tracking which jobs are in which jobs belong to which block
380	+	jobsOfBlock = {}
381	+
382		# ---- Iterate over the blocks in the dataset until ---- #
383		# ---- we've met the requested total # of events    ---- #
384		while ( (eventsRemaining > 0) and (blockCount < numBlocksInDataset) and (jobCount < totalNumberOfJobs)):
385		block = blocks[blockCount]
386		blockCount += 1
387	<
387	>	if block not in jobsOfBlock.keys() :
388	>	jobsOfBlock[block] = []
389	>
390		if self.eventsbyblock.has_key(block) :
391		numEventsInBlock = self.eventsbyblock[block]
392		common.logger.debug(5,'Events in Block File '+str(numEventsInBlock))
393	<
393	>
394		files = self.filesbyblock[block]
395		numFilesInBlock = len(files)
396		if (numFilesInBlock <= 0):
#	Line 420 | Line 398 | class Cmssw(JobType):
398		fileCount = 0
399
400		# ---- New block => New job ---- #
401	<	parString = "\\{"
401	>	parString = ""
402		# counter for number of events in files currently worked on
403		filesEventCount = 0
404		# flag if next while loop should touch new file
405		newFile = 1
406		# job event counter
407		jobSkipEventCount = 0
408	<
408	>
409		# ---- Iterate over the files in the block until we've met the requested ---- #
410		# ---- total # of events or we've gone over all the files in this block  ---- #
411		while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) and (jobCount < totalNumberOfJobs) ):
#	Line 443 | Line 421 | class Cmssw(JobType):
421		newFile = 0
422		except KeyError:
423		common.logger.message("File "+str(file)+" has unknown number of events: skipping")
446	–
424
425	+	eventsPerJobRequested = min(eventsPerJobRequested, eventsRemaining)
426		# if less events in file remain than eventsPerJobRequested
427	<	if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested ) :
427	>	if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested):
428		# if last file in block
429		if ( fileCount == numFilesInBlock-1 ) :
430		# end job using last file, use remaining events in block
431		# close job and touch new file
432		fullString = parString[:-2]
455	–	fullString += '\\}'
433		list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
434		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(filesEventCount - jobSkipEventCount)+" events (last file in block).")
435		self.jobDestination.append(blockSites[block])
436		common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
437	+	# fill jobs of block dictionary
438	+	jobsOfBlock[block].append(jobCount+1)
439		# reset counter
440		jobCount = jobCount + 1
441		totalEventCount = totalEventCount + filesEventCount - jobSkipEventCount
442		eventsRemaining = eventsRemaining - filesEventCount + jobSkipEventCount
443		jobSkipEventCount = 0
444		# reset file
445	<	parString = "\\{"
445	>	parString = ""
446		filesEventCount = 0
447		newFile = 1
448		fileCount += 1
#	Line 475 | Line 454 | class Cmssw(JobType):
454		elif ( filesEventCount - jobSkipEventCount == eventsPerJobRequested ) :
455		# close job and touch new file
456		fullString = parString[:-2]
478	–	fullString += '\\}'
457		list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
458		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
459		self.jobDestination.append(blockSites[block])
460		common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
461	+	jobsOfBlock[block].append(jobCount+1)
462		# reset counter
463		jobCount = jobCount + 1
464		totalEventCount = totalEventCount + eventsPerJobRequested
465		eventsRemaining = eventsRemaining - eventsPerJobRequested
466		jobSkipEventCount = 0
467		# reset file
468	<	parString = "\\{"
468	>	parString = ""
469		filesEventCount = 0
470		newFile = 1
471		fileCount += 1
472	<
472	>
473		# if more events in file remain than eventsPerJobRequested
474		else :
475		# close job but don't touch new file
476		fullString = parString[:-2]
498	–	fullString += '\\}'
477		list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
478		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
479		self.jobDestination.append(blockSites[block])
480		common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
481	+	jobsOfBlock[block].append(jobCount+1)
482		# increase counter
483		jobCount = jobCount + 1
484		totalEventCount = totalEventCount + eventsPerJobRequested
#	Line 509 | Line 488 | class Cmssw(JobType):
488		jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file])
489		# remove all but the last file
490		filesEventCount = self.eventsbyfile[file]
491	<	parString = "\\{"
513	<	parString += '\\\"' + file + '\\\"\,'
491	>	parString = '\\\"' + file + '\\\"\,'
492		pass # END if
493		pass # END while (iterate over files in the block)
494		pass # END while (iterate over blocks in the dataset)
495		self.ncjobs = self.total_number_of_jobs = jobCount
496		if (eventsRemaining > 0 and jobCount < totalNumberOfJobs ):
497		common.logger.message("Could not run on all requested events because some blocks not hosted at allowed sites.")
498	<	common.logger.message("\n"+str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n")
499	<
498	>	common.logger.message(str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n")
499	>
500	>	# screen output
501	>	screenOutput = "List of jobs and available destination sites:\n\n"
502	>
503	>	# keep trace of block with no sites to print a warning at the end
504	>	noSiteBlock = []
505	>	bloskNoSite = []
506	>
507	>	blockCounter = 0
508	>	for block in blocks:
509	>	if block in jobsOfBlock.keys() :
510	>	blockCounter += 1
511	>	screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),
512	>	','.join(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)))
513	>	if len(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)) == 0:
514	>	noSiteBlock.append( spanRanges(jobsOfBlock[block]) )
515	>	bloskNoSite.append( blockCounter )
516	>
517	>	common.logger.message(screenOutput)
518	>	if len(noSiteBlock) > 0 and len(bloskNoSite) > 0:
519	>	msg = 'WARNING: No sites are hosting any part of data for block:\n                '
520	>	virgola = ""
521	>	if len(bloskNoSite) > 1:
522	>	virgola = ","
523	>	for block in bloskNoSite:
524	>	msg += ' ' + str(block) + virgola
525	>	msg += '\n               Related jobs:\n                 '
526	>	virgola = ""
527	>	if len(noSiteBlock) > 1:
528	>	virgola = ","
529	>	for range_jobs in noSiteBlock:
530	>	msg += str(range_jobs) + virgola
531	>	msg += '\n               will not be submitted and this block of data can not be analyzed!\n'
532	>	if self.cfg_params.has_key('EDG.se_white_list'):
533	>	msg += 'WARNING: SE White List: '+self.cfg_params['EDG.se_white_list']+'\n'
534	>	msg += '(Hint: By whitelisting you force the job to run at this particular site(s).\n'
535	>	msg += 'Please check if the dataset is available at this site!)\n'
536	>	if self.cfg_params.has_key('EDG.ce_white_list'):
537	>	msg += 'WARNING: CE White List: '+self.cfg_params['EDG.ce_white_list']+'\n'
538	>	msg += '(Hint: By whitelisting you force the job to run at this particular site(s).\n'
539	>	msg += 'Please check if the dataset is available at this site!)\n'
540	>
541	>	common.logger.message(msg)
542	>
543		self.list_of_args = list_of_lists
544		return
545
#	Line 527 | Line 548 | class Cmssw(JobType):
548		Perform job splitting based on number of event per job
549		"""
550		common.logger.debug(5,'Splitting per events')
551	<	common.logger.message('Required '+str(self.eventsPerJob)+' events per job ')
552	<	common.logger.message('Required '+str(self.theNumberOfJobs)+' jobs in total ')
553	<	common.logger.message('Required '+str(self.total_number_of_events)+' events in total ')
551	>
552	>	if (self.selectEventsPerJob):
553	>	common.logger.message('Required '+str(self.eventsPerJob)+' events per job ')
554	>	if (self.selectNumberOfJobs):
555	>	common.logger.message('Required '+str(self.theNumberOfJobs)+' jobs in total ')
556	>	if (self.selectTotalNumberEvents):
557	>	common.logger.message('Required '+str(self.total_number_of_events)+' events in total ')
558
559		if (self.total_number_of_events < 0):
560		msg='Cannot split jobs per Events with "-1" as total number of events'
#	Line 538 | Line 563 | class Cmssw(JobType):
563		if (self.selectEventsPerJob):
564		if (self.selectTotalNumberEvents):
565		self.total_number_of_jobs = int(self.total_number_of_events/self.eventsPerJob)
566	<	elif(self.selectNumberOfJobs) :
566	>	elif(self.selectNumberOfJobs) :
567		self.total_number_of_jobs =self.theNumberOfJobs
568	<	self.total_number_of_events =int(self.theNumberOfJobs*self.eventsPerJob)
568	>	self.total_number_of_events =int(self.theNumberOfJobs*self.eventsPerJob)
569
570		elif (self.selectNumberOfJobs) :
571		self.total_number_of_jobs = self.theNumberOfJobs
572		self.eventsPerJob = int(self.total_number_of_events/self.total_number_of_jobs)
573	<
573	>
574		common.logger.debug(5,'N jobs  '+str(self.total_number_of_jobs))
575
576		# is there any remainder?
#	Line 561 | Line 586 | class Cmssw(JobType):
586		self.list_of_args = []
587		for i in range(self.total_number_of_jobs):
588		## Since there is no input, any site is good
589	<	# self.jobDestination.append(["Any"])
590	<	self.jobDestination.append([""]) #must be empty to write correctly the xml
566	<	args=''
589	>	self.jobDestination.append([""]) #must be empty to write correctly the xml
590	>	args=[]
591		if (self.firstRun):
592	<	## pythia first run
593	<	#self.list_of_args.append([(str(self.firstRun)+str(i))])
594	<	args=args+(str(self.firstRun)+str(i))
571	<	else:
572	<	## no first run
573	<	#self.list_of_args.append([str(i)])
574	<	args=args+str(i)
575	<	if (self.sourceSeed):
576	<	if (self.sourceSeedVtx):
577	<	## pythia + vtx random seed
578	<	#self.list_of_args.append([
579	<	#                          str(self.sourceSeed)+str(i),
580	<	#                          str(self.sourceSeedVtx)+str(i)
581	<	#                          ])
582	<	args=args+str(',')+str(self.sourceSeed)+str(i)+str(',')+str(self.sourceSeedVtx)+str(i)
583	<	else:
584	<	## only pythia random seed
585	<	#self.list_of_args.append([(str(self.sourceSeed)+str(i))])
586	<	args=args +str(',')+str(self.sourceSeed)+str(i)
587	<	else:
588	<	## no random seed
589	<	if str(args)=='': args=args+(str(self.firstRun)+str(i))
590	<	arguments=args.split(',')
591	<	if len(arguments)==3:self.list_of_args.append([str(arguments[0]),str(arguments[1]),str(arguments[2])])
592	<	elif len(arguments)==2:self.list_of_args.append([str(arguments[0]),str(arguments[1])])
593	<	else :self.list_of_args.append([str(arguments[0])])
594	<
595	<	#   print self.list_of_args
592	>	## pythia first run
593	>	args.append(str(self.firstRun)+str(i))
594	>	self.list_of_args.append(args)
595
596		return
597
#	Line 621 | Line 620 | class Cmssw(JobType):
620		return
621
622		def split(self, jobParams):
623	<
625	<	common.jobDB.load()
623	>
624		#### Fabio
625		njobs = self.total_number_of_jobs
626		arglist = self.list_of_args
627		# create the empty structure
628		for i in range(njobs):
629		jobParams.append("")
630	<
630	>
631	>	listID=[]
632	>	listField=[]
633		for job in range(njobs):
634		jobParams[job] = arglist[job]
635	<	# print str(arglist[job])
636	<	# print jobParams[job]
637	<	common.jobDB.setArguments(job, jobParams[job])
638	<	common.logger.debug(5,"Job "+str(job)+" Destination: "+str(self.jobDestination[job]))
639	<	common.jobDB.setDestination(job, self.jobDestination[job])
635	>	listID.append(job+1)
636	>	job_ToSave ={}
637	>	concString = ' '
638	>	argu=''
639	>	if len(jobParams[job]):
640	>	argu +=   concString.join(jobParams[job] )
641	>	job_ToSave['arguments']= str(job+1)+' '+argu## new BL--DS
642	>	job_ToSave['dlsDestination']= self.jobDestination[job]## new BL--DS
643	>	#common._db.updateJob_(job,job_ToSave)## new BL--DS
644	>	listField.append(job_ToSave)
645	>	msg="Job "+str(job)+" Arguments:   "+str(job+1)+" "+argu+"\n"  \
646	>	+"                     Destination: "+str(self.jobDestination[job])
647	>	common.logger.debug(5,msg)
648	>	#common.logger.debug(5,"Job "+str(job)+" Destination: "+str(self.jobDestination[job]))
649	>	common._db.updateJob_(listID,listField)## new BL--DS
650	>	## Pay Attention Here....DS--BL
651	>	self.argsList = (len(jobParams[0])+1)
652
641	–	common.jobDB.save()
653		return
654	<
644	<	def getJobTypeArguments(self, nj, sched):
645	<	result = ''
646	<	for i in common.jobDB.arguments(nj):
647	<	result=result+str(i)+" "
648	<	return result
649	<
654	>
655		def numberOfJobs(self):
656		# Fabio
657		return self.total_number_of_jobs
#	Line 655 | Line 660 | class Cmssw(JobType):
660		"""
661		Return the TarBall with lib and exe
662		"""
663	<
663	>
664		# if it exist, just return it
665		#
666		# Marco. Let's start to use relative path for Boss XML files
#	Line 678 | Line 683 | class Cmssw(JobType):
683		# print "swVersion = ", swVersion
684		swReleaseTop = self.scram.getReleaseTop_()
685		#print "swReleaseTop = ", swReleaseTop
686	<
686	>
687		## check if working area is release top
688		if swReleaseTop == '' or swArea == swReleaseTop:
689	+	common.logger.debug(3,"swArea = "+swArea+" swReleaseTop ="+swReleaseTop)
690		return
691
692		import tarfile
693		try: # create tar ball
694		tar = tarfile.open(self.tgzNameWithPath, "w:gz")
695		## First find the executable
696	<	if (executable != ''):
696	>	if (self.executable != ''):
697		exeWithPath = self.scram.findFile_(executable)
698		if ( not exeWithPath ):
699		raise CrabException('User executable '+executable+' not found')
700	<
700	>
701		## then check if it's private or not
702		if exeWithPath.find(swReleaseTop) == -1:
703		# the exe is private, so we must ship
704		common.logger.debug(5,"Exe "+exeWithPath+" to be tarred")
705		path = swArea+'/'
706	<	exe = string.replace(exeWithPath, path,'')
707	<	tar.add(path+exe,executable)
706	>	# distinguish case when script is in user project area or given by full path somewhere else
707	>	if exeWithPath.find(path) >= 0 :
708	>	exe = string.replace(exeWithPath, path,'')
709	>	tar.add(path+exe,exe)
710	>	else :
711	>	tar.add(exeWithPath,os.path.basename(executable))
712		pass
713		else:
714		# the exe is from release, we'll find it on WN
715		pass
716	<
716	>
717		## Now get the libraries: only those in local working area
718		libDir = 'lib'
719		lib = swArea+'/' +libDir
720		common.logger.debug(5,"lib "+lib+" to be tarred")
721		if os.path.exists(lib):
722		tar.add(lib,libDir)
723	<
723	>
724		## Now check if module dir is present
725		moduleDir = 'module'
726		module = swArea + '/' + moduleDir
#	Line 719 | Line 729 | class Cmssw(JobType):
729
730		## Now check if any data dir(s) is present
731		swAreaLen=len(swArea)
732	+	self.dataExist = False
733		for root, dirs, files in os.walk(swArea):
734		if "data" in dirs:
735	+	self.dataExist=True
736		common.logger.debug(5,"data "+root+"/data"+" to be tarred")
737		tar.add(root+"/data",root[swAreaLen:]+"/data")
738
739	<	## Add ProdAgent dir to tar
740	<	paDir = 'ProdAgentApi'
741	<	pa = os.environ['CRABDIR'] + '/' + 'ProdAgentApi'
742	<	if os.path.isdir(pa):
743	<	tar.add(pa,paDir)
744	<
739	>	### CMSSW ParameterSet
740	>	if not self.pset is None:
741	>	cfg_file = common.work_space.jobDir()+self.configFilename()
742	>	tar.add(cfg_file,self.configFilename())
743	>	common.logger.debug(5,"File added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
744	>
745	>
746	>	## Add ProdCommon dir to tar
747	>	prodcommonDir = 'ProdCommon'
748	>	prodcommonPath = os.environ['CRABDIR'] + '/' + 'ProdCommon'
749	>	if os.path.isdir(prodcommonPath):
750	>	tar.add(prodcommonPath,prodcommonDir)
751	>	common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
752	>
753	>	##### ML stuff
754	>	ML_file_list=['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py']
755	>	path=os.environ['CRABDIR'] + '/python/'
756	>	for file in ML_file_list:
757	>	tar.add(path+file,file)
758		common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
759	+
760	+	##### Utils
761	+	Utils_file_list=['parseCrabFjr.py','writeCfg.py', 'JobReportErrorCode.py']
762	+	for file in Utils_file_list:
763	+	tar.add(path+file,file)
764	+	common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
765	+
766	+	##### AdditionalFiles
767	+	for file in self.additional_inbox_files:
768	+	tar.add(file,string.split(file,'/')[-1])
769	+	common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
770	+
771		tar.close()
772		except :
773		raise CrabException('Could not create tar-ball')
#	Line 741 | Line 778 | class Cmssw(JobType):
778		raise CrabException('Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' + str(self.MaxTarBallSize) + ' MB input sandbox limit and not supported by the used GRID submission system. Please make sure that no unnecessary files are in all data directories in your local CMSSW project area as they are automatically packed into the input sandbox.')
779
780		## create tar-ball with ML stuff
781	<	self.MLtgzfile =  common.work_space.pathForTgz()+'share/MLfiles.tgz'
782	<	try:
746	<	tar = tarfile.open(self.MLtgzfile, "w:gz")
747	<	path=os.environ['CRABDIR'] + '/python/'
748	<	for file in ['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py', 'parseCrabFjr.py']:
749	<	tar.add(path+file,file)
750	<	common.logger.debug(5,"Files added to "+self.MLtgzfile+" : "+str(tar.getnames()))
751	<	tar.close()
752	<	except :
753	<	raise CrabException('Could not create ML files tar-ball')
754	<
755	<	return
756	<
757	<	def wsSetupEnvironment(self, nj):
781	>
782	>	def wsSetupEnvironment(self, nj=0):
783		"""
784		Returns part of a job script which prepares
785		the execution environment for the job 'nj'.
786		"""
787	+	if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
788	+	psetName = 'pset.py'
789	+	else:
790	+	psetName = 'pset.cfg'
791		# Prepare JobType-independent part
792	<	txt = ''
793	<
794	<	## OLI_Daniele at this level  middleware already known
766	<
767	<	txt += 'if [ $middleware == LCG ]; then \n'
792	>	txt = '\n#Written by cms_cmssw::wsSetupEnvironment\n'
793	>	txt += 'echo ">>> setup environment"\n'
794	>	txt += 'if [ $middleware == LCG ]; then \n'
795		txt += self.wsSetupCMSLCGEnvironment_()
796		txt += 'elif [ $middleware == OSG ]; then\n'
797		txt += '    WORKING_DIR=`/bin/mktemp  -d $OSG_WN_TMP/cms_XXXXXXXXXXXX`\n'
798	<	txt += '    echo "Created working directory: $WORKING_DIR"\n'
799	<	txt += '    if [ ! -d $WORKING_DIR ] ;then\n'
800	<	txt += '        echo "SET_CMS_ENV 10016 ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
801	<	txt += '        echo "JOB_EXIT_STATUS = 10016"\n'
775	<	txt += '        echo "JobExitCode=10016" | tee -a $RUNTIME_AREA/$repo\n'
776	<	txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
777	<	txt += '        rm -f $RUNTIME_AREA/$repo \n'
778	<	txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
779	<	txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
780	<	txt += '        exit 1\n'
798	>	txt += '    if [ ! $? == 0 ] ;then\n'
799	>	txt += '        echo "ERROR ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
800	>	txt += '        job_exit_code=10016\n'
801	>	txt += '        func_exit\n'
802		txt += '    fi\n'
803	+	txt += '    echo ">>> Created working directory: $WORKING_DIR"\n'
804		txt += '\n'
805		txt += '    echo "Change to working directory: $WORKING_DIR"\n'
806		txt += '    cd $WORKING_DIR\n'
807	<	txt += self.wsSetupCMSOSGEnvironment_()
807	>	txt += '    echo ">>> current directory (WORKING_DIR): $WORKING_DIR"\n'
808	>	txt += self.wsSetupCMSOSGEnvironment_()
809	>	#txt += '    echo "### Set SCRAM ARCH to ' + self.executable_arch + ' ###"\n'
810	>	#txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
811		txt += 'fi\n'
812
813		# Prepare JobType-specific part
814		scram = self.scram.commandName()
815		txt += '\n\n'
816	<	txt += 'echo "### SPECIFIC JOB SETUP ENVIRONMENT ###"\n'
816	>	txt += 'echo ">>> specific cmssw setup environment:"\n'
817	>	txt += 'echo "CMSSW_VERSION =  '+self.version+'"\n'
818		txt += scram+' project CMSSW '+self.version+'\n'
819		txt += 'status=$?\n'
820		txt += 'if [ $status != 0 ] ; then\n'
821	<	txt += '   echo "SET_EXE_ENV 10034 ==>ERROR CMSSW '+self.version+' not found on `hostname`" \n'
822	<	txt += '   echo "JOB_EXIT_STATUS = 10034"\n'
823	<	txt += '   echo "JobExitCode=10034" | tee -a $RUNTIME_AREA/$repo\n'
798	<	txt += '   dumpStatus $RUNTIME_AREA/$repo\n'
799	<	txt += '   rm -f $RUNTIME_AREA/$repo \n'
800	<	txt += '   echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
801	<	txt += '   echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
802	<	## OLI_Daniele
803	<	txt += '    if [ $middleware == OSG ]; then \n'
804	<	txt += '        echo "Remove working directory: $WORKING_DIR"\n'
805	<	txt += '        cd $RUNTIME_AREA\n'
806	<	txt += '        /bin/rm -rf $WORKING_DIR\n'
807	<	txt += '        if [ -d $WORKING_DIR ] ;then\n'
808	<	txt += '            echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
809	<	txt += '            echo "JOB_EXIT_STATUS = 10018"\n'
810	<	txt += '            echo "JobExitCode=10018" | tee -a $RUNTIME_AREA/$repo\n'
811	<	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
812	<	txt += '            rm -f $RUNTIME_AREA/$repo \n'
813	<	txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
814	<	txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
815	<	txt += '        fi\n'
816	<	txt += '    fi \n'
817	<	txt += '   exit 1 \n'
821	>	txt += '    echo "ERROR ==> CMSSW '+self.version+' not found on `hostname`" \n'
822	>	txt += '    job_exit_code=10034\n'
823	>	txt += '    func_exit\n'
824		txt += 'fi \n'
819	–	txt += 'echo "CMSSW_VERSION =  '+self.version+'"\n'
820	–	txt += 'export SCRAM_ARCH='+self.executable_arch+'\n'
825		txt += 'cd '+self.version+'\n'
826	<	### needed grep for bug in scramv1 ###
827	<	txt += scram+' runtime -sh\n'
826	>	txt += 'SOFTWARE_DIR=`pwd`\n'
827	>	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
828		txt += 'eval `'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME`\n'
829	<	txt += 'echo $PATH\n'
830	<
829	>	txt += 'if [ $? != 0 ] ; then\n'
830	>	txt += '    echo "ERROR ==> Problem with the command: "\n'
831	>	txt += '    echo "eval \`'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME \` at `hostname`"\n'
832	>	txt += '    job_exit_code=10034\n'
833	>	txt += '    func_exit\n'
834	>	txt += 'fi \n'
835		# Handle the arguments:
836		txt += "\n"
837		txt += "## number of arguments (first argument always jobnumber)\n"
838		txt += "\n"
839	<	#        txt += "narg=$#\n"
832	<	txt += "if [ $nargs -lt 2 ]\n"
839	>	txt += "if [ $nargs -lt "+str(self.argsList)+" ]\n"
840		txt += "then\n"
841	<	txt += "    echo 'SET_EXE_ENV 1 ==> ERROR Too few arguments' +$nargs+ \n"
842	<	txt += '    echo "JOB_EXIT_STATUS = 50113"\n'
843	<	txt += '    echo "JobExitCode=50113" | tee -a $RUNTIME_AREA/$repo\n'
837	<	txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
838	<	txt += '    rm -f $RUNTIME_AREA/$repo \n'
839	<	txt += '    echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
840	<	txt += '    echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
841	<	## OLI_Daniele
842	<	txt += '    if [ $middleware == OSG ]; then \n'
843	<	txt += '        echo "Remove working directory: $WORKING_DIR"\n'
844	<	txt += '        cd $RUNTIME_AREA\n'
845	<	txt += '        /bin/rm -rf $WORKING_DIR\n'
846	<	txt += '        if [ -d $WORKING_DIR ] ;then\n'
847	<	txt += '            echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
848	<	txt += '            echo "JOB_EXIT_STATUS = 50114"\n'
849	<	txt += '            echo "JobExitCode=50114" | tee -a $RUNTIME_AREA/$repo\n'
850	<	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
851	<	txt += '            rm -f $RUNTIME_AREA/$repo \n'
852	<	txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
853	<	txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
854	<	txt += '        fi\n'
855	<	txt += '    fi \n'
856	<	txt += "    exit 1\n"
841	>	txt += "    echo 'ERROR ==> Too few arguments' +$nargs+ \n"
842	>	txt += '    job_exit_code=50113\n'
843	>	txt += "    func_exit\n"
844		txt += "fi\n"
845		txt += "\n"
846
847		# Prepare job-specific part
848		job = common.job_list[nj]
849	<	if self.pset != None: #CarlosDaniele
849	>	### FEDE FOR DBS OUTPUT PUBLICATION
850	>	if (self.datasetPath):
851	>	txt += '\n'
852	>	txt += 'DatasetPath='+self.datasetPath+'\n'
853	>
854	>	datasetpath_split = self.datasetPath.split("/")
855	>
856	>	txt += 'PrimaryDataset='+datasetpath_split[1]+'\n'
857	>	txt += 'DataTier='+datasetpath_split[2]+'\n'
858	>	txt += 'ApplicationFamily=cmsRun\n'
859	>
860	>	else:
861	>	txt += 'DatasetPath=MCDataTier\n'
862	>	txt += 'PrimaryDataset=null\n'
863	>	txt += 'DataTier=null\n'
864	>	txt += 'ApplicationFamily=MCDataTier\n'
865	>	if self.pset != None:
866		pset = os.path.basename(job.configFilename())
867		txt += '\n'
868	+	txt += 'cp  $RUNTIME_AREA/'+pset+' .\n'
869		if (self.datasetPath): # standard job
870	<	#txt += 'InputFiles=$2\n'
871	<	txt += 'InputFiles=${args[1]}\n'
872	<	txt += 'MaxEvents=${args[2]}\n'
869	<	txt += 'SkipEvents=${args[3]}\n'
870	>	txt += 'InputFiles=${args[1]}; export InputFiles\n'
871	>	txt += 'MaxEvents=${args[2]}; export MaxEvents\n'
872	>	txt += 'SkipEvents=${args[3]}; export SkipEvents\n'
873		txt += 'echo "Inputfiles:<$InputFiles>"\n'
871	–	txt += 'sed "s#{\'INPUT\'}#$InputFiles#" $RUNTIME_AREA/'+pset+' > pset_tmp_1.cfg\n'
874		txt += 'echo "MaxEvents:<$MaxEvents>"\n'
873	–	txt += 'sed "s#INPUTMAXEVENTS#$MaxEvents#" pset_tmp_1.cfg > pset_tmp_2.cfg\n'
875		txt += 'echo "SkipEvents:<$SkipEvents>"\n'
875	–	txt += 'sed "s#INPUTSKIPEVENTS#$SkipEvents#" pset_tmp_2.cfg > pset.cfg\n'
876		else:  # pythia like job
877	<	if (self.sourceSeed):
878	<	txt += 'FirstRun=${args[1]}\n'
877	>	txt += 'PreserveSeeds='  + ','.join(self.preserveSeeds)  + '; export PreserveSeeds\n'
878	>	txt += 'IncrementSeeds=' + ','.join(self.incrementSeeds) + '; export IncrementSeeds\n'
879	>	txt += 'echo "PreserveSeeds: <$PreserveSeeds>"\n'
880	>	txt += 'echo "IncrementSeeds:<$IncrementSeeds>"\n'
881	>	if (self.firstRun):
882	>	txt += 'FirstRun=${args[1]}; export FirstRun\n'
883		txt += 'echo "FirstRun: <$FirstRun>"\n'
880	–	txt += 'sed "s#\<INPUTFIRSTRUN\>#$FirstRun#" $RUNTIME_AREA/'+pset+' > tmp_1.cfg\n'
881	–	else:
882	–	txt += '# Copy untouched pset\n'
883	–	txt += 'cp $RUNTIME_AREA/'+pset+' tmp_1.cfg\n'
884	–	if (self.sourceSeed):
885	–	#                    txt += 'Seed=$2\n'
886	–	txt += 'Seed=${args[2]}\n'
887	–	txt += 'echo "Seed: <$Seed>"\n'
888	–	txt += 'sed "s#\<INPUT\>#$Seed#" tmp_1.cfg > tmp_2.cfg\n'
889	–	if (self.sourceSeedVtx):
890	–	#                        txt += 'VtxSeed=$3\n'
891	–	txt += 'VtxSeed=${args[3]}\n'
892	–	txt += 'echo "VtxSeed: <$VtxSeed>"\n'
893	–	txt += 'sed "s#INPUTVTX#$VtxSeed#" tmp_2.cfg > pset.cfg\n'
894	–	else:
895	–	txt += 'mv tmp_2.cfg pset.cfg\n'
896	–	else:
897	–	txt += 'mv tmp_1.cfg pset.cfg\n'
898	–	# txt += '# Copy untouched pset\n'
899	–	# txt += 'cp $RUNTIME_AREA/'+pset+' pset.cfg\n'
884
885	+	txt += 'mv -f ' + pset + ' ' + psetName + '\n'
886
902	–	if len(self.additional_inbox_files) > 0:
903	–	for file in self.additional_inbox_files:
904	–	relFile = file.split("/")[-1]
905	–	txt += 'if [ -e $RUNTIME_AREA/'+relFile+' ] ; then\n'
906	–	txt += '   cp $RUNTIME_AREA/'+relFile+' .\n'
907	–	txt += '   chmod +x '+relFile+'\n'
908	–	txt += 'fi\n'
909	–	pass
887
888	<	if self.pset != None: #CarlosDaniele
889	<	txt += 'echo "### END JOB SETUP ENVIRONMENT ###"\n\n'
890	<
888	>	if self.pset != None:
889	>	# FUTURE: Can simply for 2_1_x and higher
890	>	txt += '\n'
891	>	txt += 'echo "***** cat ' + psetName + ' *********"\n'
892	>	txt += 'cat ' + psetName + '\n'
893	>	txt += 'echo "****** end ' + psetName + ' ********"\n'
894		txt += '\n'
895	<	txt += 'echo "***** cat pset.cfg *********"\n'
896	<	txt += 'cat pset.cfg\n'
917	<	txt += 'echo "****** end pset.cfg ********"\n'
895	>	txt += 'PSETHASH=`edmConfigHash < ' + psetName + '` \n'
896	>	txt += 'echo "PSETHASH = $PSETHASH" \n'
897		txt += '\n'
919	–	# txt += 'echo "***** cat pset1.cfg *********"\n'
920	–	# txt += 'cat pset1.cfg\n'
921	–	# txt += 'echo "****** end pset1.cfg ********"\n'
898		return txt
899
900	<	def wsBuildExe(self, nj=0):
900	>	def wsUntarSoftware(self, nj=0):
901		"""
902		Put in the script the commands to build an executable
903		or a library.
904		"""
905
906	<	txt = ""
906	>	txt = '\n#Written by cms_cmssw::wsUntarSoftware\n'
907
908		if os.path.isfile(self.tgzNameWithPath):
909	<	txt += 'echo "tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'"\n'
909	>	txt += 'echo ">>> tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+' :" \n'
910		txt += 'tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
911	+	txt += 'ls -Al \n'
912		txt += 'untar_status=$? \n'
913		txt += 'if [ $untar_status -ne 0 ]; then \n'
914	<	txt += '   echo "SET_EXE 1 ==> ERROR Untarring .tgz file failed"\n'
915	<	txt += '   echo "JOB_EXIT_STATUS = $untar_status" \n'
916	<	txt += '   echo "JobExitCode=$untar_status" | tee -a $RUNTIME_AREA/$repo\n'
940	<	txt += '   if [ $middleware == OSG ]; then \n'
941	<	txt += '       echo "Remove working directory: $WORKING_DIR"\n'
942	<	txt += '       cd $RUNTIME_AREA\n'
943	<	txt += '       /bin/rm -rf $WORKING_DIR\n'
944	<	txt += '       if [ -d $WORKING_DIR ] ;then\n'
945	<	txt += '           echo "SET_EXE 50999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Untarring .tgz file failed"\n'
946	<	txt += '           echo "JOB_EXIT_STATUS = 50999"\n'
947	<	txt += '           echo "JobExitCode=50999" | tee -a $RUNTIME_AREA/$repo\n'
948	<	txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
949	<	txt += '           rm -f $RUNTIME_AREA/$repo \n'
950	<	txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
951	<	txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
952	<	txt += '       fi\n'
953	<	txt += '   fi \n'
954	<	txt += '   \n'
955	<	txt += '   exit 1 \n'
914	>	txt += '   echo "ERROR ==> Untarring .tgz file failed"\n'
915	>	txt += '   job_exit_code=$untar_status\n'
916	>	txt += '   func_exit\n'
917		txt += 'else \n'
918		txt += '   echo "Successful untar" \n'
919		txt += 'fi \n'
920		txt += '\n'
921	<	txt += 'echo "Include ProdAgentApi in PYTHONPATH"\n'
921	>	txt += 'echo ">>> Include ProdCommon in PYTHONPATH:"\n'
922		txt += 'if [ -z "$PYTHONPATH" ]; then\n'
923	<	txt += '   export PYTHONPATH=ProdAgentApi\n'
923	>	txt += '   export PYTHONPATH=$RUNTIME_AREA/ProdCommon\n'
924		txt += 'else\n'
925	<	txt += '   export PYTHONPATH=ProdAgentApi:${PYTHONPATH}\n'
925	>	txt += '   export PYTHONPATH=$RUNTIME_AREA/ProdCommon:${PYTHONPATH}\n'
926	>	txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
927		txt += 'fi\n'
928		txt += '\n'
929
930		pass
931	<
931	>
932	>	return txt
933	>
934	>	def wsBuildExe(self, nj=0):
935	>	"""
936	>	Put in the script the commands to build an executable
937	>	or a library.
938	>	"""
939	>
940	>	txt = '\n#Written by cms_cmssw::wsBuildExe\n'
941	>	txt += 'echo ">>> moving CMSSW software directories in `pwd`" \n'
942	>
943	>	txt += 'rm -r lib/ module/ \n'
944	>	txt += 'mv $RUNTIME_AREA/lib/ . \n'
945	>	txt += 'mv $RUNTIME_AREA/module/ . \n'
946	>	if self.dataExist == True: txt += 'mv $RUNTIME_AREA/src/ . \n'
947	>	if len(self.additional_inbox_files)>0:
948	>	for file in self.additional_inbox_files:
949	>	txt += 'mv $RUNTIME_AREA/'+file+' . \n'
950	>	txt += 'mv $RUNTIME_AREA/ProdCommon/ . \n'
951	>
952	>	txt += 'if [ -z "$PYTHONPATH" ]; then\n'
953	>	txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdCommon\n'
954	>	txt += 'else\n'
955	>	txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdCommon:${PYTHONPATH}\n'
956	>	txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
957	>	txt += 'fi\n'
958	>	txt += '\n'
959	>
960		return txt
961
962		def modifySteeringCards(self, nj):
963		"""
964	<	modify the card provided by the user,
964	>	modify the card provided by the user,
965		writing a new card into share dir
966		"""
967	<
967	>
968		def executableName(self):
969		if self.scriptExe: #CarlosDaniele
970		return "sh "
#	Line 982 | Line 972 | class Cmssw(JobType):
972		return self.executable
973
974		def executableArgs(self):
975	+	# FUTURE: This function tests the CMSSW version. Can be simplified as we drop support for old versions
976		if self.scriptExe:#CarlosDaniele
977		return   self.scriptExe + " $NJob"
978	<	else:
979	<	return " -p pset.cfg"
978	>	else:
979	>	ex_args = ""
980	>	# FUTURE: This tests the CMSSW version. Can remove code as versions deprecated
981	>	# Framework job report
982	>	if (self.CMSSW_major >= 1 and self.CMSSW_minor >= 5) or (self.CMSSW_major >= 2):
983	>	ex_args += " -j $RUNTIME_AREA/crab_fjr_$NJob.xml"
984	>	# Type of config file
985	>	if self.CMSSW_major >= 2 :
986	>	ex_args += " -p pset.py"
987	>	else:
988	>	ex_args += " -p pset.cfg"
989	>	return ex_args
990
991		def inputSandbox(self, nj):
992		"""
#	Line 997 | Line 998 | class Cmssw(JobType):
998		## code
999		if os.path.isfile(self.tgzNameWithPath):
1000		inp_box.append(self.tgzNameWithPath)
1001	<	if os.path.isfile(self.MLtgzfile):
1002	<	inp_box.append(self.MLtgzfile)
1002	<	## config
1003	<	if not self.pset is None:
1004	<	inp_box.append(common.work_space.pathForTgz() + 'job/' + self.configFilename())
1005	<	## additional input files
1006	<	for file in self.additional_inbox_files:
1007	<	inp_box.append(file)
1001	>	wrapper = os.path.basename(str(common._db.queryTask('scriptName')))
1002	>	inp_box.append(common.work_space.pathForTgz() +'job/'+ wrapper)
1003		return inp_box
1004
1005		def outputSandbox(self, nj):
#	Line 1015 | Line 1010 | class Cmssw(JobType):
1010
1011		## User Declared output files
1012		for out in (self.output_file+self.output_file_sandbox):
1013	<	n_out = nj + 1
1013	>	n_out = nj + 1
1014		out_box.append(self.numberFile_(out,str(n_out)))
1015		return out_box
1016
#	Line 1030 | Line 1025 | class Cmssw(JobType):
1025		Returns part of a job script which renames the produced files.
1026		"""
1027
1028	<	txt = '\n'
1029	<	txt += '# directory content\n'
1028	>	txt = '\n#Written by cms_cmssw::wsRenameOutput\n'
1029	>	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
1030	>	txt += 'echo ">>> current directory content:"\n'
1031		txt += 'ls \n'
1032	+	txt += '\n'
1033
1034	<	for fileWithSuffix in (self.output_file+self.output_file_sandbox):
1034	>	for fileWithSuffix in (self.output_file):
1035		output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1036		txt += '\n'
1037		txt += '# check output file\n'
1038	<	txt += 'ls '+fileWithSuffix+'\n'
1039	<	txt += 'ls_result=$?\n'
1040	<	txt += 'if [ $ls_result -ne 0 ] ; then\n'
1041	<	txt += '   echo "ERROR: Problem with output file"\n'
1042	<	if common.scheduler.boss_scheduler_name == 'condor_g':
1038	>	txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1039	>	if (self.copy_data == 1):  # For OSG nodes, file is in $WORKING_DIR, should not be moved to $RUNTIME_AREA
1040	>	txt += '    mv '+fileWithSuffix+' '+output_file_num+'\n'
1041	>	txt += '    ln -s `pwd`/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1042	>	else:
1043	>	txt += '    mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1044	>	txt += '    ln -s $RUNTIME_AREA/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1045	>	txt += 'else\n'
1046	>	txt += '    job_exit_code=60302\n'
1047	>	txt += '    echo "WARNING: Output file '+fileWithSuffix+' not found"\n'
1048	>	if common.scheduler.name().upper() == 'CONDOR_G':
1049		txt += '    if [ $middleware == OSG ]; then \n'
1050		txt += '        echo "prepare dummy output file"\n'
1051		txt += '        echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
1052		txt += '    fi \n'
1050	–	txt += 'else\n'
1051	–	txt += '   cp '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1053		txt += 'fi\n'
1054	<
1055	<	txt += 'cd $RUNTIME_AREA\n'
1056	<	txt += 'cd $RUNTIME_AREA\n'
1056	<	### OLI_DANIELE
1057	<	txt += 'if [ $middleware == OSG ]; then\n'
1058	<	txt += '    cd $RUNTIME_AREA\n'
1059	<	txt += '    echo "Remove working directory: $WORKING_DIR"\n'
1060	<	txt += '    /bin/rm -rf $WORKING_DIR\n'
1061	<	txt += '    if [ -d $WORKING_DIR ] ;then\n'
1062	<	txt += '        echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1063	<	txt += '        echo "JOB_EXIT_STATUS = 60999"\n'
1064	<	txt += '        echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
1065	<	txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
1066	<	txt += '        rm -f $RUNTIME_AREA/$repo \n'
1067	<	txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1068	<	txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1069	<	txt += '    fi\n'
1070	<	txt += 'fi\n'
1071	<	txt += '\n'
1072	<
1073	<	file_list = ''
1074	<	## Add to filelist only files to be possibly copied to SE
1075	<	for fileWithSuffix in self.output_file:
1076	<	output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1077	<	file_list=file_list+output_file_num+' '
1078	<	file_list=file_list[:-1]
1079	<	txt += 'file_list="'+file_list+'"\n'
1054	>	file_list = []
1055	>	for fileWithSuffix in (self.output_file):
1056	>	file_list.append(self.numberFile_(fileWithSuffix, '$NJob'))
1057
1058	+	txt += 'file_list="'+string.join(file_list,' ')+'"\n'
1059	+	txt += '\n'
1060	+	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
1061	+	txt += 'echo ">>> current directory content:"\n'
1062	+	txt += 'ls \n'
1063	+	txt += '\n'
1064	+	txt += 'cd $RUNTIME_AREA\n'
1065	+	txt += 'echo ">>> current directory (RUNTIME_AREA):  $RUNTIME_AREA"\n'
1066		return txt
1067
1068		def numberFile_(self, file, txt):
#	Line 1088 | Line 1073 | class Cmssw(JobType):
1073		# take away last extension
1074		name = p[0]
1075		for x in p[1:-1]:
1076	<	name=name+"."+x
1076	>	name=name+"."+x
1077		# add "_txt"
1078		if len(p)>1:
1079	<	ext = p[len(p)-1]
1080	<	result = name + '_' + txt + "." + ext
1079	>	ext = p[len(p)-1]
1080	>	result = name + '_' + txt + "." + ext
1081		else:
1082	<	result = name + '_' + txt
1083	<
1082	>	result = name + '_' + txt
1083	>
1084		return result
1085
1086		def getRequirements(self, nj=[]):
1087		"""
1088	<	return job requirements to add to jdl files
1088	>	return job requirements to add to jdl files
1089		"""
1090		req = ''
1091		if self.version:
1092		req='Member("VO-cms-' + \
1093		self.version + \
1094		'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1095	+	## SL add requirement for OS version only if SL4
1096	+	#reSL4 = re.compile( r'slc4' )
1097	+	if self.executable_arch: # and reSL4.search(self.executable_arch):
1098	+	req+=' && Member("VO-cms-' + \
1099	+	self.executable_arch + \
1100	+	'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1101
1102		req = req + ' && (other.GlueHostNetworkAdapterOutboundIP)'
1103	+	if common.scheduler.name() == "glitecoll":
1104	+	req += ' && other.GlueCEStateStatus == "Production" '
1105
1106		return req
1107
1108		def configFilename(self):
1109		""" return the config filename """
1110	<	return self.name()+'.cfg'
1110	>	# FUTURE: Can remove cfg mode for CMSSW >= 2_1_x
1111	>	if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
1112	>	return self.name()+'.py'
1113	>	else:
1114	>	return self.name()+'.cfg'
1115
1119	–	### OLI_DANIELE
1116		def wsSetupCMSOSGEnvironment_(self):
1117		"""
1118		Returns part of a job script which is prepares
1119		the execution environment and which is common for all CMS jobs.
1120		"""
1121	<	txt = '\n'
1122	<	txt += '   echo "### SETUP CMS OSG  ENVIRONMENT ###"\n'
1123	<	txt += '   if [ -f $GRID3_APP_DIR/cmssoft/cmsset_default.sh ] ;then\n'
1124	<	txt += '      # Use $GRID3_APP_DIR/cmssoft/cmsset_default.sh to setup cms software\n'
1125	<	txt += '       source $GRID3_APP_DIR/cmssoft/cmsset_default.sh '+self.version+'\n'
1126	<	txt += '   elif [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n'
1121	>	txt = '\n#Written by cms_cmssw::wsSetupCMSOSGEnvironment_\n'
1122	>	txt += '    echo ">>> setup CMS OSG environment:"\n'
1123	>	txt += '    echo "set SCRAM ARCH to ' + self.executable_arch + '"\n'
1124	>	txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
1125	>	txt += '    echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
1126	>	txt += '    if [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n'
1127		txt += '      # Use $OSG_APP/cmssoft/cms/cmsset_default.sh to setup cms software\n'
1128	<	txt += '       source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1129	<	txt += '   else\n'
1130	<	txt += '       echo "SET_CMS_ENV 10020 ==> ERROR $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1131	<	txt += '       echo "JOB_EXIT_STATUS = 10020"\n'
1132	<	txt += '       echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
1133	<	txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
1138	<	txt += '       rm -f $RUNTIME_AREA/$repo \n'
1139	<	txt += '       echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1140	<	txt += '       echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1141	<	txt += '       exit 1\n'
1142	<	txt += '\n'
1143	<	txt += '       echo "Remove working directory: $WORKING_DIR"\n'
1144	<	txt += '       cd $RUNTIME_AREA\n'
1145	<	txt += '       /bin/rm -rf $WORKING_DIR\n'
1146	<	txt += '       if [ -d $WORKING_DIR ] ;then\n'
1147	<	txt += '            echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1148	<	txt += '            echo "JOB_EXIT_STATUS = 10017"\n'
1149	<	txt += '            echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n'
1150	<	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
1151	<	txt += '            rm -f $RUNTIME_AREA/$repo \n'
1152	<	txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1153	<	txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1154	<	txt += '       fi\n'
1155	<	txt += '\n'
1156	<	txt += '       exit 1\n'
1157	<	txt += '   fi\n'
1128	>	txt += '        source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1129	>	txt += '    else\n'
1130	>	txt += '        echo "ERROR ==> $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1131	>	txt += '        job_exit_code=10020\n'
1132	>	txt += '        func_exit\n'
1133	>	txt += '    fi\n'
1134		txt += '\n'
1135	<	txt += '   echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1136	<	txt += '   echo " END SETUP CMS OSG  ENVIRONMENT "\n'
1135	>	txt += '    echo "==> setup cms environment ok"\n'
1136	>	txt += '    echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
1137
1138		return txt
1139	<
1139	>
1140		### OLI_DANIELE
1141		def wsSetupCMSLCGEnvironment_(self):
1142		"""
1143		Returns part of a job script which is prepares
1144		the execution environment and which is common for all CMS jobs.
1145		"""
1146	<	txt  = '   \n'
1147	<	txt += '   echo " ### SETUP CMS LCG  ENVIRONMENT ### "\n'
1148	<	txt += '   if [ ! $VO_CMS_SW_DIR ] ;then\n'
1149	<	txt += '       echo "SET_CMS_ENV 10031 ==> ERROR CMS software dir not found on WN `hostname`"\n'
1150	<	txt += '       echo "JOB_EXIT_STATUS = 10031" \n'
1151	<	txt += '       echo "JobExitCode=10031" | tee -a $RUNTIME_AREA/$repo\n'
1152	<	txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
1153	<	txt += '       rm -f $RUNTIME_AREA/$repo \n'
1154	<	txt += '       echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1155	<	txt += '       echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1156	<	txt += '       exit 1\n'
1157	<	txt += '   else\n'
1158	<	txt += '       echo "Sourcing environment... "\n'
1159	<	txt += '       if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
1160	<	txt += '           echo "SET_CMS_ENV 10020 ==> ERROR cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1161	<	txt += '           echo "JOB_EXIT_STATUS = 10020"\n'
1162	<	txt += '           echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
1163	<	txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1164	<	txt += '           rm -f $RUNTIME_AREA/$repo \n'
1165	<	txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1166	<	txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1167	<	txt += '           exit 1\n'
1168	<	txt += '       fi\n'
1169	<	txt += '       echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1170	<	txt += '       source $VO_CMS_SW_DIR/cmsset_default.sh\n'
1171	<	txt += '       result=$?\n'
1172	<	txt += '       if [ $result -ne 0 ]; then\n'
1173	<	txt += '           echo "SET_CMS_ENV 10032 ==> ERROR problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1174	<	txt += '           echo "JOB_EXIT_STATUS = 10032"\n'
1175	<	txt += '           echo "JobExitCode=10032" | tee -a $RUNTIME_AREA/$repo\n'
1176	<	txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1177	<	txt += '           rm -f $RUNTIME_AREA/$repo \n'
1178	<	txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1179	<	txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1180	<	txt += '           exit 1\n'
1181	<	txt += '       fi\n'
1182	<	txt += '   fi\n'
1183	<	txt += '   \n'
1184	<	txt += '   echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1185	<	txt += '   echo "### END SETUP CMS LCG ENVIRONMENT ###"\n'
1146	>	txt = '\n#Written by cms_cmssw::wsSetupCMSLCGEnvironment_\n'
1147	>	txt += '    echo ">>> setup CMS LCG environment:"\n'
1148	>	txt += '    echo "set SCRAM ARCH and BUILD_ARCH to ' + self.executable_arch + ' ###"\n'
1149	>	txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
1150	>	txt += '    export BUILD_ARCH='+self.executable_arch+'\n'
1151	>	txt += '    if [ ! $VO_CMS_SW_DIR ] ;then\n'
1152	>	txt += '        echo "ERROR ==> CMS software dir not found on WN `hostname`"\n'
1153	>	txt += '        job_exit_code=10031\n'
1154	>	txt += '        func_exit\n'
1155	>	txt += '    else\n'
1156	>	txt += '        echo "Sourcing environment... "\n'
1157	>	txt += '        if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
1158	>	txt += '            echo "ERROR ==> cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1159	>	txt += '            job_exit_code=10020\n'
1160	>	txt += '            func_exit\n'
1161	>	txt += '        fi\n'
1162	>	txt += '        echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1163	>	txt += '        source $VO_CMS_SW_DIR/cmsset_default.sh\n'
1164	>	txt += '        result=$?\n'
1165	>	txt += '        if [ $result -ne 0 ]; then\n'
1166	>	txt += '            echo "ERROR ==> problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1167	>	txt += '            job_exit_code=10032\n'
1168	>	txt += '            func_exit\n'
1169	>	txt += '        fi\n'
1170	>	txt += '    fi\n'
1171	>	txt += '    \n'
1172	>	txt += '    echo "==> setup cms environment ok"\n'
1173	>	return txt
1174	>
1175	>	### FEDE FOR DBS OUTPUT PUBLICATION
1176	>	def modifyReport(self, nj):
1177	>	"""
1178	>	insert the part of the script that modifies the FrameworkJob Report
1179	>	"""
1180	>
1181	>	txt = '\n#Written by cms_cmssw::modifyReport\n'
1182	>	publish_data = int(self.cfg_params.get('USER.publish_data',0))
1183	>	if (publish_data == 1):
1184	>	processedDataset = self.cfg_params['USER.publish_data_name']
1185	>	LFNBaseName = LFNBase(processedDataset)
1186	>
1187	>	txt += 'if [ $copy_exit_status -eq 0 ]; then\n'
1188	>	txt += '    FOR_LFN=%s_${PSETHASH}/\n'%(LFNBaseName)
1189	>	txt += 'else\n'
1190	>	txt += '    FOR_LFN=/copy_problems/ \n'
1191	>	txt += '    SE=""\n'
1192	>	txt += '    SE_PATH=""\n'
1193	>	txt += 'fi\n'
1194	>
1195	>	txt += 'echo ">>> Modify Job Report:" \n'
1196	>	txt += 'chmod a+x $SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py\n'
1197	>	txt += 'ProcessedDataset='+processedDataset+'\n'
1198	>	txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1199	>	txt += 'echo "SE = $SE"\n'
1200	>	txt += 'echo "SE_PATH = $SE_PATH"\n'
1201	>	txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1202	>	txt += 'echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n'
1203	>	txt += 'echo "$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1204	>	txt += '$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1205	>	txt += 'modifyReport_result=$?\n'
1206	>	txt += 'if [ $modifyReport_result -ne 0 ]; then\n'
1207	>	txt += '    modifyReport_result=70500\n'
1208	>	txt += '    job_exit_code=$modifyReport_result\n'
1209	>	txt += '    echo "ModifyReportResult=$modifyReport_result" | tee -a $RUNTIME_AREA/$repo\n'
1210	>	txt += '    echo "WARNING: Problem with ModifyJobReport"\n'
1211	>	txt += 'else\n'
1212	>	txt += '    mv NewFrameworkJobReport.xml $RUNTIME_AREA/crab_fjr_$NJob.xml\n'
1213	>	txt += 'fi\n'
1214		return txt
1215
1216		def setParam_(self, param, value):
#	Line 1215 | Line 1219 | class Cmssw(JobType):
1219		def getParams(self):
1220		return self._params
1221
1218	–	def setTaskid_(self):
1219	–	self._taskId = self.cfg_params['taskId']
1220	–
1221	–	def getTaskid(self):
1222	–	return self._taskId
1223	–
1224	–	#######################################################################
1222		def uniquelist(self, old):
1223		"""
1224		remove duplicates from a list
#	Line 1230 | Line 1227 | class Cmssw(JobType):
1227		for e in old:
1228		nd[e]=0
1229		return nd.keys()
1230	+
1231	+	def outList(self):
1232	+	"""
1233	+	check the dimension of the output files
1234	+	"""
1235	+	txt = ''
1236	+	txt += 'echo ">>> list of expected files on output sandbox"\n'
1237	+	listOutFiles = []
1238	+	stdout = 'CMSSW_$NJob.stdout'
1239	+	stderr = 'CMSSW_$NJob.stderr'
1240	+	if (self.return_data == 1):
1241	+	for file in (self.output_file+self.output_file_sandbox):
1242	+	listOutFiles.append(self.numberFile_(file, '$NJob'))
1243	+	listOutFiles.append(stdout)
1244	+	listOutFiles.append(stderr)
1245	+	else:
1246	+	for file in (self.output_file_sandbox):
1247	+	listOutFiles.append(self.numberFile_(file, '$NJob'))
1248	+	listOutFiles.append(stdout)
1249	+	listOutFiles.append(stderr)
1250	+	txt += 'echo "output files: '+string.join(listOutFiles,' ')+'"\n'
1251	+	txt += 'filesToCheck="'+string.join(listOutFiles,' ')+'"\n'
1252	+	txt += 'export filesToCheck\n'
1253	+	return txt

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines