ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/cms_cmssw.py

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.73 by gutsche, Sun Apr 8 18:39:51 2007 UTC vs.
Revision 1.265 by ewv, Tue Dec 16 17:07:08 2008 UTC

#	Line 2 | Line 2 | from JobType import JobType
2		from crab_logger import Logger
3		from crab_exceptions import *
4		from crab_util import *
5	+	from WMCore.SiteScreening.BlackWhiteListParser import SEBlackWhiteListParser
6		import common
6	–	import PsetManipulator
7	–	import DataDiscovery
8	–	import DataDiscovery_DBS2
9	–	import DataLocation
7		import Scram
8
9	<	import os, string, re, shutil, glob
9	>	import os, string, glob
10
11		class Cmssw(JobType):
12	<	def __init__(self, cfg_params, ncjobs):
12	>	def __init__(self, cfg_params, ncjobs,skip_blocks, isNew):
13		JobType.__init__(self, 'CMSSW')
14		common.logger.debug(3,'CMSSW::__init__')
15	+	self.skip_blocks = skip_blocks
16	+	self.argsList = []
17
19	–	# Marco.
18		self._params = {}
19		self.cfg_params = cfg_params
20
21	<	try:
22	<	self.MaxTarBallSize = float(self.cfg_params['EDG.maxtarballsize'])
23	<	except KeyError:
24	<	self.MaxTarBallSize = 100.0
21	>	# init BlackWhiteListParser
22	>	seWhiteList = cfg_params.get('EDG.se_white_list',[])
23	>	seBlackList = cfg_params.get('EDG.se_black_list',[])
24	>	self.blackWhiteListParser = SEBlackWhiteListParser(seWhiteList, seBlackList, common.logger)
25	>
26	>	### Temporary patch to automatically skip the ISB size check:
27	>	server=self.cfg_params.get('CRAB.server_name',None)
28	>	size = 9.5
29	>	if server or common.scheduler.name().upper() in ['LSF','CAF']: size = 99999
30	>	### D.S.
31	>	self.MaxTarBallSize = float(self.cfg_params.get('EDG.maxtarballsize',size))
32
33		# number of jobs requested to be created, limit obj splitting
34		self.ncjobs = ncjobs
35
36		log = common.logger
37	<
37	>
38		self.scram = Scram.Scram(cfg_params)
39		self.additional_inbox_files = []
40		self.scriptExe = ''
#	Line 37 | Line 42 | class Cmssw(JobType):
42		self.executable_arch = self.scram.getArch()
43		self.tgz_name = 'default.tgz'
44		self.scriptName = 'CMSSW.sh'
45	<	self.pset = ''      #scrip use case Da
46	<	self.datasetPath = '' #scrip use case Da
45	>	self.pset = ''
46	>	self.datasetPath = ''
47
48		# set FJR file name
49		self.fjrFileName = 'crab_fjr.xml'
50
51		self.version = self.scram.getSWVersion()
52	<	common.taskDB.setDict('codeVersion',self.version)
53	<	self.setParam_('application', self.version)
52	>	version_array = self.version.split('_')
53	>	self.CMSSW_major = 0
54	>	self.CMSSW_minor = 0
55	>	self.CMSSW_patch = 0
56	>	try:
57	>	self.CMSSW_major = int(version_array[1])
58	>	self.CMSSW_minor = int(version_array[2])
59	>	self.CMSSW_patch = int(version_array[3])
60	>	except:
61	>	msg = "Cannot parse CMSSW version string: " + self.version + " for major and minor release number!"
62	>	raise CrabException(msg)
63
64		### collect Data cards
65
66	<	## get DBS mode
67	<	try:
54	<	self.use_dbs_2 = int(self.cfg_params['CMSSW.use_dbs_2'])
55	<	except KeyError:
56	<	self.use_dbs_2 = 0
57	<
58	<	try:
59	<	tmp =  cfg_params['CMSSW.datasetpath']
60	<	log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
61	<	if string.lower(tmp)=='none':
62	<	self.datasetPath = None
63	<	self.selectNoInput = 1
64	<	else:
65	<	self.datasetPath = tmp
66	<	self.selectNoInput = 0
67	<	except KeyError:
68	<	msg = "Error: datasetpath not defined "
66	>	if not cfg_params.has_key('CMSSW.datasetpath'):
67	>	msg = "Error: datasetpath not defined "
68		raise CrabException(msg)
69
70	<	# ML monitoring
71	<	# split dataset path style: /PreProdR3Minbias/SIM/GEN-SIM
73	<	if not self.datasetPath:
74	<	self.setParam_('dataset', 'None')
75	<	self.setParam_('owner', 'None')
76	<	else:
77	<	datasetpath_split = self.datasetPath.split("/")
78	<	self.setParam_('dataset', datasetpath_split[1])
79	<	self.setParam_('owner', datasetpath_split[-1])
70	>	### Temporary: added to remove input file control in the case of PU
71	>	self.dataset_pu = cfg_params.get('CMSSW.dataset_pu', None)
72
73	<	self.setTaskid_()
74	<	self.setParam_('taskId', self.cfg_params['taskId'])
73	>	tmp =  cfg_params['CMSSW.datasetpath']
74	>	log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
75
76	<	self.dataTiers = []
76	>	if tmp =='':
77	>	msg = "Error: datasetpath not defined "
78	>	raise CrabException(msg)
79	>	elif string.lower(tmp)=='none':
80	>	self.datasetPath = None
81	>	self.selectNoInput = 1
82	>	else:
83	>	self.datasetPath = tmp
84	>	self.selectNoInput = 0
85
86	+	self.dataTiers = []
87	+	self.debugWrap = ''
88	+	self.debug_wrapper = cfg_params.get('USER.debug_wrapper',False)
89	+	if self.debug_wrapper: self.debugWrap='--debug'
90		## now the application
91	<	try:
92	<	self.executable = cfg_params['CMSSW.executable']
93	<	self.setParam_('exe', self.executable)
94	<	log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable)
91	<	msg = "Default executable cmsRun overridden. Switch to " + self.executable
92	<	log.debug(3,msg)
93	<	except KeyError:
94	<	self.executable = 'cmsRun'
95	<	self.setParam_('exe', self.executable)
96	<	msg = "User executable not defined. Use cmsRun"
97	<	log.debug(3,msg)
98	<	pass
91	>	self.managedGenerators = ['madgraph','comphep']
92	>	self.generator = cfg_params.get('CMSSW.generator','pythia').lower()
93	>	self.executable = cfg_params.get('CMSSW.executable','cmsRun')
94	>	log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable)
95
96	<	try:
101	<	self.pset = cfg_params['CMSSW.pset']
102	<	log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset)
103	<	if self.pset.lower() != 'none' :
104	<	if (not os.path.exists(self.pset)):
105	<	raise CrabException("User defined PSet file "+self.pset+" does not exist")
106	<	else:
107	<	self.pset = None
108	<	except KeyError:
96	>	if not cfg_params.has_key('CMSSW.pset'):
97		raise CrabException("PSet file missing. Cannot run cmsRun ")
98	+	self.pset = cfg_params['CMSSW.pset']
99	+	log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset)
100	+	if self.pset.lower() != 'none' :
101	+	if (not os.path.exists(self.pset)):
102	+	raise CrabException("User defined PSet file "+self.pset+" does not exist")
103	+	else:
104	+	self.pset = None
105
106		# output files
107		## stuff which must be returned always via sandbox
#	Line 116 | Line 111 | class Cmssw(JobType):
111		self.output_file_sandbox.append(self.fjrFileName)
112
113		# other output files to be returned via sandbox or copied to SE
114	<	try:
115	<	self.output_file = []
116	<	tmp = cfg_params['CMSSW.output_file']
117	<	if tmp != '':
118	<	tmpOutFiles = string.split(cfg_params['CMSSW.output_file'],',')
119	<	log.debug(7, 'cmssw::cmssw(): output files '+str(tmpOutFiles))
120	<	for tmp in tmpOutFiles:
121	<	tmp=string.strip(tmp)
127	<	self.output_file.append(tmp)
128	<	pass
129	<	else:
130	<	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available")
131	<	pass
132	<	pass
133	<	except KeyError:
134	<	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available")
135	<	pass
114	>	outfileflag = False
115	>	self.output_file = []
116	>	tmp = cfg_params.get('CMSSW.output_file',None)
117	>	if tmp :
118	>	self.output_file = [x.strip() for x in tmp.split(',')]
119	>	outfileflag = True #output found
120	>	#else:
121	>	#    log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
122
123		# script_exe file as additional file in inputSandbox
124	<	try:
125	<	self.scriptExe = cfg_params['USER.script_exe']
126	<	if self.scriptExe != '':
127	<	if not os.path.isfile(self.scriptExe):
128	<	msg ="ERROR. file "+self.scriptExe+" not found"
129	<	raise CrabException(msg)
144	<	self.additional_inbox_files.append(string.strip(self.scriptExe))
145	<	except KeyError:
146	<	self.scriptExe = ''
124	>	self.scriptExe = cfg_params.get('USER.script_exe',None)
125	>	if self.scriptExe :
126	>	if not os.path.isfile(self.scriptExe):
127	>	msg ="ERROR. file "+self.scriptExe+" not found"
128	>	raise CrabException(msg)
129	>	self.additional_inbox_files.append(string.strip(self.scriptExe))
130
148	–	#CarlosDaniele
131		if self.datasetPath == None and self.pset == None and self.scriptExe == '' :
132	<	msg ="Error. script_exe  not defined"
133	<	raise CrabException(msg)
132	>	msg ="Error. script_exe  not defined"
133	>	raise CrabException(msg)
134	>
135	>	# use parent files...
136	>	self.useParent = self.cfg_params.get('CMSSW.use_parent',False)
137
138		## additional input files
139	<	try:
139	>	if cfg_params.has_key('USER.additional_input_files'):
140		tmpAddFiles = string.split(cfg_params['USER.additional_input_files'],',')
141		for tmp in tmpAddFiles:
142		tmp = string.strip(tmp)
143		dirname = ''
144		if not tmp[0]=="/": dirname = "."
145	<	files = glob.glob(os.path.join(dirname, tmp))
145	>	files = []
146	>	if string.find(tmp,"*")>-1:
147	>	files = glob.glob(os.path.join(dirname, tmp))
148	>	if len(files)==0:
149	>	raise CrabException("No additional input file found with this pattern: "+tmp)
150	>	else:
151	>	files.append(tmp)
152		for file in files:
153		if not os.path.exists(file):
154		raise CrabException("Additional input file not found: "+file)
155		pass
156	<	storedFile = common.work_space.shareDir()+file
166	<	shutil.copyfile(file, storedFile)
167	<	self.additional_inbox_files.append(string.strip(storedFile))
156	>	self.additional_inbox_files.append(string.strip(file))
157		pass
158		pass
159		common.logger.debug(5,"Additional input files: "+str(self.additional_inbox_files))
160	<	except KeyError:
172	<	pass
173	<
174	<	# files per job
175	<	try:
176	<	if (cfg_params['CMSSW.files_per_jobs']):
177	<	raise CrabException("files_per_jobs no longer supported.  Quitting.")
178	<	except KeyError:
179	<	pass
160	>	pass
161
162		## Events per job
163	<	try:
163	>	if cfg_params.has_key('CMSSW.events_per_job'):
164		self.eventsPerJob =int( cfg_params['CMSSW.events_per_job'])
165		self.selectEventsPerJob = 1
166	<	except KeyError:
166	>	else:
167		self.eventsPerJob = -1
168		self.selectEventsPerJob = 0
169	<
169	>
170		## number of jobs
171	<	try:
171	>	if cfg_params.has_key('CMSSW.number_of_jobs'):
172		self.theNumberOfJobs =int( cfg_params['CMSSW.number_of_jobs'])
173		self.selectNumberOfJobs = 1
174	<	except KeyError:
174	>	else:
175		self.theNumberOfJobs = 0
176		self.selectNumberOfJobs = 0
177
178	<	try:
178	>	if cfg_params.has_key('CMSSW.total_number_of_events'):
179		self.total_number_of_events = int(cfg_params['CMSSW.total_number_of_events'])
180		self.selectTotalNumberEvents = 1
181	<	except KeyError:
181	>	if self.selectNumberOfJobs  == 1:
182	>	if (self.total_number_of_events != -1) and int(self.total_number_of_events) < int(self.theNumberOfJobs):
183	>	msg = 'Must specify at least one event per job. total_number_of_events > number_of_jobs '
184	>	raise CrabException(msg)
185	>	else:
186		self.total_number_of_events = 0
187		self.selectTotalNumberEvents = 0
188
189	<	if self.pset != None: #CarlosDaniele
189	>	if self.pset != None:
190		if ( (self.selectTotalNumberEvents + self.selectEventsPerJob + self.selectNumberOfJobs) != 2 ):
191		msg = 'Must define exactly two of total_number_of_events, events_per_job, or number_of_jobs.'
192		raise CrabException(msg)
#	Line 210 | Line 195 | class Cmssw(JobType):
195		msg = 'Must specify  number_of_jobs.'
196		raise CrabException(msg)
197
198	<	## source seed for pythia
199	<	try:
200	<	self.sourceSeed = int(cfg_params['CMSSW.pythia_seed'])
201	<	except KeyError:
202	<	self.sourceSeed = None
203	<	common.logger.debug(5,"No seed given")
198	>	## New method of dealing with seeds
199	>	self.incrementSeeds = []
200	>	self.preserveSeeds = []
201	>	if cfg_params.has_key('CMSSW.preserve_seeds'):
202	>	tmpList = cfg_params['CMSSW.preserve_seeds'].split(',')
203	>	for tmp in tmpList:
204	>	tmp.strip()
205	>	self.preserveSeeds.append(tmp)
206	>	if cfg_params.has_key('CMSSW.increment_seeds'):
207	>	tmpList = cfg_params['CMSSW.increment_seeds'].split(',')
208	>	for tmp in tmpList:
209	>	tmp.strip()
210	>	self.incrementSeeds.append(tmp)
211	>
212	>	## FUTURE: Can remove in CRAB 2.4.0
213	>	self.sourceSeed    = cfg_params.get('CMSSW.pythia_seed',None)
214	>	self.sourceSeedVtx = cfg_params.get('CMSSW.vtx_seed',None)
215	>	self.sourceSeedG4  = cfg_params.get('CMSSW.g4_seed',None)
216	>	self.sourceSeedMix = cfg_params.get('CMSSW.mix_seed',None)
217	>	if self.sourceSeed or self.sourceSeedVtx or self.sourceSeedG4 or self.sourceSeedMix:
218	>	msg = 'pythia_seed, vtx_seed, g4_seed, and mix_seed are no longer valid settings. You must use increment_seeds or preserve_seeds'
219	>	raise CrabException(msg)
220
221	<	try:
222	<	self.sourceSeedVtx = int(cfg_params['CMSSW.vtx_seed'])
223	<	except KeyError:
224	<	self.sourceSeedVtx = None
225	<	common.logger.debug(5,"No vertex seed given")
225	<	try:
226	<	self.firstRun = int(cfg_params['CMSSW.first_run'])
227	<	except KeyError:
228	<	self.firstRun = None
229	<	common.logger.debug(5,"No first run given")
230	<	if self.pset != None: #CarlosDaniele
231	<	self.PsetEdit = PsetManipulator.PsetManipulator(self.pset) #Daniele Pset
221	>	self.firstRun = cfg_params.get('CMSSW.first_run',None)
222	>
223	>	# Copy/return
224	>	self.copy_data = int(cfg_params.get('USER.copy_data',0))
225	>	self.return_data = int(cfg_params.get('USER.return_data',0))
226
227		#DBSDLS-start
228	<	## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
228	>	## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
229		self.maxEvents=0  # max events available   ( --> check the requested nb. of evts in Creator.py)
230		self.DBSPaths={}  # all dbs paths requested ( --> input to the site local discovery script)
231		self.jobDestination=[]  # Site destination(s) for each job (list of lists)
#	Line 240 | Line 234 | class Cmssw(JobType):
234		blockSites = {}
235		if self.datasetPath:
236		blockSites = self.DataDiscoveryAndLocation(cfg_params)
237	<	#DBSDLS-end
237	>	#DBSDLS-end
238
245	–	self.tgzNameWithPath = self.getTarBall(self.executable)
246	–
239		## Select Splitting
240	<	if self.selectNoInput:
241	<	if self.pset == None: #CarlosDaniele
240	>	if self.selectNoInput:
241	>	if self.pset == None:
242		self.jobSplittingForScript()
243		else:
244		self.jobSplittingNoInput()
245	<	else:
245	>	elif (cfg_params.get('CMSSW.noblockboundary',0)):
246	>	self.jobSplittingNoBlockBoundary(blockSites)
247	>	else:
248		self.jobSplittingByBlocks(blockSites)
249
250	<	# modify Pset
251	<	if self.pset != None: #CarlosDaniele
252	<	try:
253	<	if (self.datasetPath): # standard job
254	<	# allow to processa a fraction of events in a file
255	<	self.PsetEdit.inputModule("INPUT")
256	<	self.PsetEdit.maxEvent("INPUTMAXEVENTS")
257	<	self.PsetEdit.skipEvent("INPUTSKIPEVENTS")
258	<	else:  # pythia like job
259	<	self.PsetEdit.maxEvent(self.eventsPerJob)
260	<	if (self.firstRun):
261	<	self.PsetEdit.pythiaFirstRun("INPUTFIRSTRUN")  #First Run
262	<	if (self.sourceSeed) :
263	<	self.PsetEdit.pythiaSeed("INPUT")
264	<	if (self.sourceSeedVtx) :
265	<	self.PsetEdit.pythiaSeedVtx("INPUTVTX")
266	<	# add FrameworkJobReport to parameter-set
267	<	self.PsetEdit.addCrabFJR(self.fjrFileName)
268	<	self.PsetEdit.psetWriter(self.configFilename())
269	<	except:
270	<	msg='Error while manipuliating ParameterSet: exiting...'
271	<	raise CrabException(msg)
250	>	# modify Pset only the first time
251	>	if isNew:
252	>	if self.pset != None:
253	>	import PsetManipulator as pp
254	>	PsetEdit = pp.PsetManipulator(self.pset)
255	>	try:
256	>	# Add FrameworkJobReport to parameter-set, set max events.
257	>	# Reset later for data jobs by writeCFG which does all modifications
258	>	PsetEdit.addCrabFJR(self.fjrFileName) # FUTURE: Job report addition not needed by CMSSW>1.5
259	>	PsetEdit.maxEvent(self.eventsPerJob)
260	>	PsetEdit.skipEvent(0)
261	>	PsetEdit.psetWriter(self.configFilename())
262	>	## If present, add TFileService to output files
263	>	if not int(cfg_params.get('CMSSW.skip_TFileService_output',0)):
264	>	tfsOutput = PsetEdit.getTFileService()
265	>	if tfsOutput:
266	>	if tfsOutput in self.output_file:
267	>	common.logger.debug(5,"Output from TFileService "+tfsOutput+" already in output files")
268	>	else:
269	>	outfileflag = True #output found
270	>	self.output_file.append(tfsOutput)
271	>	common.logger.message("Adding "+tfsOutput+" to output files (from TFileService)")
272	>	pass
273	>	pass
274	>	## If present and requested, add PoolOutputModule to output files
275	>	if int(cfg_params.get('CMSSW.get_edm_output',0)):
276	>	edmOutput = PsetEdit.getPoolOutputModule()
277	>	if edmOutput:
278	>	if edmOutput in self.output_file:
279	>	common.logger.debug(5,"Output from PoolOutputModule "+edmOutput+" already in output files")
280	>	else:
281	>	self.output_file.append(edmOutput)
282	>	common.logger.message("Adding "+edmOutput+" to output files (from PoolOutputModule)")
283	>	pass
284	>	pass
285	>	except CrabException:
286	>	msg='Error while manipulating ParameterSet: exiting...'
287	>	raise CrabException(msg)
288	>	## Prepare inputSandbox TarBall (only the first time)
289	>	self.tgzNameWithPath = self.getTarBall(self.executable)
290
291		def DataDiscoveryAndLocation(self, cfg_params):
292
293	+	import DataDiscovery
294	+	import DataLocation
295		common.logger.debug(10,"CMSSW::DataDiscoveryAndLocation()")
296
297		datasetPath=self.datasetPath
298
299		## Contact the DBS
300	<	common.logger.message("Contacting DBS...")
300	>	common.logger.message("Contacting Data Discovery Services ...")
301		try:
302	<
289	<	if self.use_dbs_2 == 1 :
290	<	self.pubdata=DataDiscovery_DBS2.DataDiscovery_DBS2(datasetPath, cfg_params)
291	<	else :
292	<	self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params)
302	>	self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params,self.skip_blocks)
303		self.pubdata.fetchDBSInfo()
304
305		except DataDiscovery.NotExistingDatasetError, ex :
#	Line 301 | Line 311 | class Cmssw(JobType):
311		except DataDiscovery.DataDiscoveryError, ex:
312		msg = 'ERROR ***: failed Data Discovery in DBS :  %s'%ex.getErrorMessage()
313		raise CrabException(msg)
304	–	except DataDiscovery_DBS2.NotExistingDatasetError_DBS2, ex :
305	–	msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
306	–	raise CrabException(msg)
307	–	except DataDiscovery_DBS2.NoDataTierinProvenanceError_DBS2, ex :
308	–	msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
309	–	raise CrabException(msg)
310	–	except DataDiscovery_DBS2.DataDiscoveryError_DBS2, ex:
311	–	msg = 'ERROR ***: failed Data Discovery in DBS :  %s'%ex.getErrorMessage()
312	–	raise CrabException(msg)
313	–
314	–	## get list of all required data in the form of dbs paths  (dbs path = /dataset/datatier/owner)
315	–	common.logger.message("Required data are :"+self.datasetPath)
314
315		self.filesbyblock=self.pubdata.getFiles()
316		self.eventsbyblock=self.pubdata.getEventsPerBlock()
317		self.eventsbyfile=self.pubdata.getEventsPerFile()
318	+	self.parentFiles=self.pubdata.getParent()
319
320		## get max number of events
321	<	self.maxEvents=self.pubdata.getMaxEvents() ##  self.maxEvents used in Creator.py
323	<	common.logger.message("The number of available events is %s\n"%self.maxEvents)
321	>	self.maxEvents=self.pubdata.getMaxEvents()
322
325	–	common.logger.message("Contacting DLS...")
323		## Contact the DLS and build a list of sites hosting the fileblocks
324		try:
325		dataloc=DataLocation.DataLocation(self.filesbyblock.keys(),cfg_params)
326		dataloc.fetchDLSInfo()
327	+
328		except DataLocation.DataLocationError , ex:
329		msg = 'ERROR ***: failed Data Location in DLS \n %s '%ex.getErrorMessage()
330		raise CrabException(msg)
331	<
331	>
332
333		sites = dataloc.getSites()
334	+	if len(sites)==0:
335	+	msg = 'ERROR ***: no location for any of the blocks of this dataset: \n %s \n'%datasetPath
336	+	msg += "Maybe the dataset is located only at T1's, where analysis jobs are not allowed\n"
337	+	msg += "Please check DataDiscovery page https://cmsweb.cern.ch/dbs_discovery/\n"
338	+	raise CrabException(msg)
339	+
340		allSites = []
341		listSites = sites.values()
342		for listSite in listSites:
#	Line 340 | Line 344 | class Cmssw(JobType):
344		allSites.append(oneSite)
345		allSites = self.uniquelist(allSites)
346
347	<	common.logger.message("Sites ("+str(len(allSites))+") hosting part/all of dataset: "+str(allSites))
348	<	common.logger.debug(6, "List of Sites: "+str(allSites))
347	>	# screen output
348	>	common.logger.message("Requested dataset: " + datasetPath + " has " + str(self.maxEvents) + " events in " + str(len(self.filesbyblock.keys())) + " blocks.\n")
349	>
350		return sites
351	<
351	>
352		def jobSplittingByBlocks(self, blockSites):
353		"""
354		Perform job splitting. Jobs run over an integer number of files
#	Line 393 | Line 398 | class Cmssw(JobType):
398		totalNumberOfJobs = 999999999
399		else :
400		totalNumberOfJobs = self.ncjobs
396	–
401
402		blocks = blockSites.keys()
403		blockCount = 0
#	Line 403 | Line 407 | class Cmssw(JobType):
407		jobCount = 0
408		list_of_lists = []
409
410	+	# list tracking which jobs are in which jobs belong to which block
411	+	jobsOfBlock = {}
412	+
413		# ---- Iterate over the blocks in the dataset until ---- #
414		# ---- we've met the requested total # of events    ---- #
415		while ( (eventsRemaining > 0) and (blockCount < numBlocksInDataset) and (jobCount < totalNumberOfJobs)):
416		block = blocks[blockCount]
417		blockCount += 1
418	<
418	>	if block not in jobsOfBlock.keys() :
419	>	jobsOfBlock[block] = []
420	>
421		if self.eventsbyblock.has_key(block) :
422		numEventsInBlock = self.eventsbyblock[block]
423		common.logger.debug(5,'Events in Block File '+str(numEventsInBlock))
424	<
424	>
425		files = self.filesbyblock[block]
426		numFilesInBlock = len(files)
427		if (numFilesInBlock <= 0):
#	Line 420 | Line 429 | class Cmssw(JobType):
429		fileCount = 0
430
431		# ---- New block => New job ---- #
432	<	parString = "\\{"
432	>	parString = ""
433		# counter for number of events in files currently worked on
434		filesEventCount = 0
435		# flag if next while loop should touch new file
436		newFile = 1
437		# job event counter
438		jobSkipEventCount = 0
439	<
439	>
440		# ---- Iterate over the files in the block until we've met the requested ---- #
441		# ---- total # of events or we've gone over all the files in this block  ---- #
442	+	pString=''
443		while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) and (jobCount < totalNumberOfJobs) ):
444		file = files[fileCount]
445	+	if self.useParent:
446	+	parent = self.parentFiles[file]
447	+	for f in parent :
448	+	pString += '\\\"' + f + '\\\"\,'
449	+	common.logger.debug(6, "File "+str(file)+" has the following parents: "+str(parent))
450	+	common.logger.write("File "+str(file)+" has the following parents: "+str(parent))
451		if newFile :
452		try:
453		numEventsInFile = self.eventsbyfile[file]
#	Line 443 | Line 459 | class Cmssw(JobType):
459		newFile = 0
460		except KeyError:
461		common.logger.message("File "+str(file)+" has unknown number of events: skipping")
446	–
462
463	+	eventsPerJobRequested = min(eventsPerJobRequested, eventsRemaining)
464		# if less events in file remain than eventsPerJobRequested
465	<	if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested ) :
465	>	if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested):
466		# if last file in block
467		if ( fileCount == numFilesInBlock-1 ) :
468		# end job using last file, use remaining events in block
469		# close job and touch new file
470		fullString = parString[:-2]
471	<	fullString += '\\}'
472	<	list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
471	>	if self.useParent:
472	>	fullParentString = pString[:-2]
473	>	list_of_lists.append([fullString,fullParentString,str(-1),str(jobSkipEventCount)])
474	>	else:
475	>	list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
476		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(filesEventCount - jobSkipEventCount)+" events (last file in block).")
477		self.jobDestination.append(blockSites[block])
478		common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
479	+	# fill jobs of block dictionary
480	+	jobsOfBlock[block].append(jobCount+1)
481		# reset counter
482		jobCount = jobCount + 1
483		totalEventCount = totalEventCount + filesEventCount - jobSkipEventCount
484		eventsRemaining = eventsRemaining - filesEventCount + jobSkipEventCount
485		jobSkipEventCount = 0
486		# reset file
487	<	parString = "\\{"
487	>	pString = ""
488	>	parString = ""
489		filesEventCount = 0
490		newFile = 1
491		fileCount += 1
#	Line 475 | Line 497 | class Cmssw(JobType):
497		elif ( filesEventCount - jobSkipEventCount == eventsPerJobRequested ) :
498		# close job and touch new file
499		fullString = parString[:-2]
500	<	fullString += '\\}'
501	<	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
500	>	if self.useParent:
501	>	fullParentString = pString[:-2]
502	>	list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount)])
503	>	else:
504	>	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
505	>	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
506	>	self.jobDestination.append(blockSites[block])
507	>	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
508	>	jobsOfBlock[block].append(jobCount+1)
509	>	# reset counter
510	>	jobCount = jobCount + 1
511	>	totalEventCount = totalEventCount + eventsPerJobRequested
512	>	eventsRemaining = eventsRemaining - eventsPerJobRequested
513	>	jobSkipEventCount = 0
514	>	# reset file
515	>	pString = ""
516	>	parString = ""
517	>	filesEventCount = 0
518	>	newFile = 1
519	>	fileCount += 1
520	>
521	>	# if more events in file remain than eventsPerJobRequested
522	>	else :
523	>	# close job but don't touch new file
524	>	fullString = parString[:-2]
525	>	if self.useParent:
526	>	fullParentString = pString[:-2]
527	>	list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount)])
528	>	else:
529	>	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
530	>	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
531	>	self.jobDestination.append(blockSites[block])
532	>	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
533	>	jobsOfBlock[block].append(jobCount+1)
534	>	# increase counter
535	>	jobCount = jobCount + 1
536	>	totalEventCount = totalEventCount + eventsPerJobRequested
537	>	eventsRemaining = eventsRemaining - eventsPerJobRequested
538	>	# calculate skip events for last file
539	>	# use filesEventCount (contains several files), jobSkipEventCount and eventsPerJobRequest
540	>	jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file])
541	>	# remove all but the last file
542	>	filesEventCount = self.eventsbyfile[file]
543	>	if self.useParent:
544	>	for f in parent : pString += '\\\"' + f + '\\\"\,'
545	>	parString = '\\\"' + file + '\\\"\,'
546	>	pass # END if
547	>	pass # END while (iterate over files in the block)
548	>	pass # END while (iterate over blocks in the dataset)
549	>	self.ncjobs = self.total_number_of_jobs = jobCount
550	>	if (eventsRemaining > 0 and jobCount < totalNumberOfJobs ):
551	>	common.logger.message("Could not run on all requested events because some blocks not hosted at allowed sites.")
552	>	common.logger.message(str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n")
553	>
554	>	# screen output
555	>	screenOutput = "List of jobs and available destination sites:\n\n"
556	>
557	>	# keep trace of block with no sites to print a warning at the end
558	>	noSiteBlock = []
559	>	bloskNoSite = []
560	>
561	>	blockCounter = 0
562	>	for block in blocks:
563	>	if block in jobsOfBlock.keys() :
564	>	blockCounter += 1
565	>	screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),
566	>	','.join(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)))
567	>	if len(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)) == 0:
568	>	noSiteBlock.append( spanRanges(jobsOfBlock[block]) )
569	>	bloskNoSite.append( blockCounter )
570	>
571	>	common.logger.message(screenOutput)
572	>	if len(noSiteBlock) > 0 and len(bloskNoSite) > 0:
573	>	msg = 'WARNING: No sites are hosting any part of data for block:\n                '
574	>	virgola = ""
575	>	if len(bloskNoSite) > 1:
576	>	virgola = ","
577	>	for block in bloskNoSite:
578	>	msg += ' ' + str(block) + virgola
579	>	msg += '\n               Related jobs:\n                 '
580	>	virgola = ""
581	>	if len(noSiteBlock) > 1:
582	>	virgola = ","
583	>	for range_jobs in noSiteBlock:
584	>	msg += str(range_jobs) + virgola
585	>	msg += '\n               will not be submitted and this block of data can not be analyzed!\n'
586	>	if self.cfg_params.has_key('EDG.se_white_list'):
587	>	msg += 'WARNING: SE White List: '+self.cfg_params['EDG.se_white_list']+'\n'
588	>	msg += '(Hint: By whitelisting you force the job to run at this particular site(s).\n'
589	>	msg += 'Please check if the dataset is available at this site!)\n'
590	>	if self.cfg_params.has_key('EDG.ce_white_list'):
591	>	msg += 'WARNING: CE White List: '+self.cfg_params['EDG.ce_white_list']+'\n'
592	>	msg += '(Hint: By whitelisting you force the job to run at this particular site(s).\n'
593	>	msg += 'Please check if the dataset is available at this site!)\n'
594	>
595	>	common.logger.message(msg)
596	>
597	>	self.list_of_args = list_of_lists
598	>	return
599	>
600	>	def jobSplittingNoBlockBoundary(self,blockSites):
601	>	"""
602	>	"""
603	>	# ---- Handle the possible job splitting configurations ---- #
604	>	if (self.selectTotalNumberEvents):
605	>	totalEventsRequested = self.total_number_of_events
606	>	if (self.selectEventsPerJob):
607	>	eventsPerJobRequested = self.eventsPerJob
608	>	if (self.selectNumberOfJobs):
609	>	totalEventsRequested = self.theNumberOfJobs * self.eventsPerJob
610	>
611	>	# If user requested all the events in the dataset
612	>	if (totalEventsRequested == -1):
613	>	eventsRemaining=self.maxEvents
614	>	# If user requested more events than are in the dataset
615	>	elif (totalEventsRequested > self.maxEvents):
616	>	eventsRemaining = self.maxEvents
617	>	common.logger.message("Requested "+str(self.total_number_of_events)+ " events, but only "+str(self.maxEvents)+" events are available.")
618	>	# If user requested less events than are in the dataset
619	>	else:
620	>	eventsRemaining = totalEventsRequested
621	>
622	>	# If user requested more events per job than are in the dataset
623	>	if (self.selectEventsPerJob and eventsPerJobRequested > self.maxEvents):
624	>	eventsPerJobRequested = self.maxEvents
625	>
626	>	# For user info at end
627	>	totalEventCount = 0
628	>
629	>	if (self.selectTotalNumberEvents and self.selectNumberOfJobs):
630	>	eventsPerJobRequested = int(eventsRemaining/self.theNumberOfJobs)
631	>
632	>	if (self.selectNumberOfJobs):
633	>	common.logger.message("May not create the exact number_of_jobs requested.")
634	>
635	>	if ( self.ncjobs == 'all' ) :
636	>	totalNumberOfJobs = 999999999
637	>	else :
638	>	totalNumberOfJobs = self.ncjobs
639	>
640	>	blocks = blockSites.keys()
641	>	blockCount = 0
642	>	# Backup variable in case self.maxEvents counted events in a non-included block
643	>	numBlocksInDataset = len(blocks)
644	>
645	>	jobCount = 0
646	>	list_of_lists = []
647	>
648	>	#AF
649	>	#AF do not reset input files and event count on block boundary
650	>	#AF
651	>	parString=""
652	>	filesEventCount = 0
653	>	#AF
654	>
655	>	# list tracking which jobs are in which jobs belong to which block
656	>	jobsOfBlock = {}
657	>	while ( (eventsRemaining > 0) and (blockCount < numBlocksInDataset) and (jobCount < totalNumberOfJobs)):
658	>	block = blocks[blockCount]
659	>	blockCount += 1
660	>	if block not in jobsOfBlock.keys() :
661	>	jobsOfBlock[block] = []
662	>
663	>	if self.eventsbyblock.has_key(block) :
664	>	numEventsInBlock = self.eventsbyblock[block]
665	>	common.logger.debug(5,'Events in Block File '+str(numEventsInBlock))
666	>	files = self.filesbyblock[block]
667	>	numFilesInBlock = len(files)
668	>	if (numFilesInBlock <= 0):
669	>	continue
670	>	fileCount = 0
671	>	#AF
672	>	#AF do not reset input files and event count of block boundary
673	>	#AF
674	>	## ---- New block => New job ---- #
675	>	#parString = ""
676	>	# counter for number of events in files currently worked on
677	>	#filesEventCount = 0
678	>	#AF
679	>	# flag if next while loop should touch new file
680	>	newFile = 1
681	>	# job event counter
682	>	jobSkipEventCount = 0
683	>
684	>	# ---- Iterate over the files in the block until we've met the requested ---- #
685	>	# ---- total # of events or we've gone over all the files in this block  ---- #
686	>	pString=''
687	>	while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) and (jobCount < totalNumberOfJobs) ):
688	>	file = files[fileCount]
689	>	if self.useParent:
690	>	parent = self.parentFiles[file]
691	>	for f in parent :
692	>	pString += '\\\"' + f + '\\\"\,'
693	>	common.logger.debug(6, "File "+str(file)+" has the following parents: "+str(parent))
694	>	common.logger.write("File "+str(file)+" has the following parents: "+str(parent))
695	>	if newFile :
696	>	try:
697	>	numEventsInFile = self.eventsbyfile[file]
698	>	common.logger.debug(6, "File "+str(file)+" has "+str(numEventsInFile)+" events")
699	>	# increase filesEventCount
700	>	filesEventCount += numEventsInFile
701	>	# Add file to current job
702	>	parString += '\\\"' + file + '\\\"\,'
703	>	newFile = 0
704	>	except KeyError:
705	>	common.logger.message("File "+str(file)+" has unknown number of events: skipping")
706	>	eventsPerJobRequested = min(eventsPerJobRequested, eventsRemaining)
707	>	#common.logger.message("AF filesEventCount %s - jobSkipEventCount %s "%(filesEventCount,jobSkipEventCount))
708	>	# if less events in file remain than eventsPerJobRequested
709	>	if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested):
710	>	#AF
711	>	#AF skip fileboundary part
712	>	#AF
713	>	# go to next file
714	>	newFile = 1
715	>	fileCount += 1
716	>	# if events in file equal to eventsPerJobRequested
717	>	elif ( filesEventCount - jobSkipEventCount == eventsPerJobRequested ) :
718	>	# close job and touch new file
719	>	fullString = parString[:-2]
720	>	if self.useParent:
721	>	fullParentString = pString[:-2]
722	>	list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount)])
723	>	else:
724	>	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
725		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
726		self.jobDestination.append(blockSites[block])
727		common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
728	+	jobsOfBlock[block].append(jobCount+1)
729		# reset counter
730		jobCount = jobCount + 1
731		totalEventCount = totalEventCount + eventsPerJobRequested
732		eventsRemaining = eventsRemaining - eventsPerJobRequested
733		jobSkipEventCount = 0
734		# reset file
735	<	parString = "\\{"
735	>	pString = ""
736	>	parString = ""
737		filesEventCount = 0
738		newFile = 1
739		fileCount += 1
740	<
740	>
741		# if more events in file remain than eventsPerJobRequested
742		else :
743		# close job but don't touch new file
744		fullString = parString[:-2]
745	<	fullString += '\\}'
746	<	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
745	>	if self.useParent:
746	>	fullParentString = pString[:-2]
747	>	list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount)])
748	>	else:
749	>	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
750		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
751		self.jobDestination.append(blockSites[block])
752		common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
753	+	jobsOfBlock[block].append(jobCount+1)
754		# increase counter
755		jobCount = jobCount + 1
756		totalEventCount = totalEventCount + eventsPerJobRequested
#	Line 509 | Line 760 | class Cmssw(JobType):
760		jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file])
761		# remove all but the last file
762		filesEventCount = self.eventsbyfile[file]
763	<	parString = "\\{"
764	<	parString += '\\\"' + file + '\\\"\,'
763	>	if self.useParent:
764	>	for f in parent : pString += '\\\"' + f + '\\\"\,'
765	>	parString = '\\\"' + file + '\\\"\,'
766		pass # END if
767		pass # END while (iterate over files in the block)
768		pass # END while (iterate over blocks in the dataset)
769		self.ncjobs = self.total_number_of_jobs = jobCount
770		if (eventsRemaining > 0 and jobCount < totalNumberOfJobs ):
771	+	common.logger.message("eventsRemaining "+str(eventsRemaining))
772	+	common.logger.message("jobCount "+str(jobCount))
773	+	common.logger.message(" totalNumberOfJobs "+str(totalNumberOfJobs))
774		common.logger.message("Could not run on all requested events because some blocks not hosted at allowed sites.")
775	<	common.logger.message("\n"+str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n")
776	<
775	>	common.logger.message(str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n")
776	>
777	>	# screen output
778	>	screenOutput = "List of jobs and available destination sites:\n\n"
779	>
780	>	#AF
781	>	#AF   skip check on  block with no sites
782	>	#AF
783		self.list_of_args = list_of_lists
784	+
785		return
786
787	+
788	+
789		def jobSplittingNoInput(self):
790		"""
791		Perform job splitting based on number of event per job
792		"""
793		common.logger.debug(5,'Splitting per events')
794	<	common.logger.message('Required '+str(self.eventsPerJob)+' events per job ')
795	<	common.logger.message('Required '+str(self.theNumberOfJobs)+' jobs in total ')
796	<	common.logger.message('Required '+str(self.total_number_of_events)+' events in total ')
794	>
795	>	if (self.selectEventsPerJob):
796	>	common.logger.message('Required '+str(self.eventsPerJob)+' events per job ')
797	>	if (self.selectNumberOfJobs):
798	>	common.logger.message('Required '+str(self.theNumberOfJobs)+' jobs in total ')
799	>	if (self.selectTotalNumberEvents):
800	>	common.logger.message('Required '+str(self.total_number_of_events)+' events in total ')
801
802		if (self.total_number_of_events < 0):
803		msg='Cannot split jobs per Events with "-1" as total number of events'
#	Line 538 | Line 806 | class Cmssw(JobType):
806		if (self.selectEventsPerJob):
807		if (self.selectTotalNumberEvents):
808		self.total_number_of_jobs = int(self.total_number_of_events/self.eventsPerJob)
809	<	elif(self.selectNumberOfJobs) :
809	>	elif(self.selectNumberOfJobs) :
810		self.total_number_of_jobs =self.theNumberOfJobs
811	<	self.total_number_of_events =int(self.theNumberOfJobs*self.eventsPerJob)
811	>	self.total_number_of_events =int(self.theNumberOfJobs*self.eventsPerJob)
812
813		elif (self.selectNumberOfJobs) :
814		self.total_number_of_jobs = self.theNumberOfJobs
815		self.eventsPerJob = int(self.total_number_of_events/self.total_number_of_jobs)
816	<
816	>
817		common.logger.debug(5,'N jobs  '+str(self.total_number_of_jobs))
818
819		# is there any remainder?
#	Line 561 | Line 829 | class Cmssw(JobType):
829		self.list_of_args = []
830		for i in range(self.total_number_of_jobs):
831		## Since there is no input, any site is good
832	<	# self.jobDestination.append(["Any"])
833	<	self.jobDestination.append([""]) #must be empty to write correctly the xml
566	<	args=''
832	>	self.jobDestination.append([""]) #must be empty to write correctly the xml
833	>	args=[]
834		if (self.firstRun):
835	<	## pythia first run
836	<	#self.list_of_args.append([(str(self.firstRun)+str(i))])
837	<	args=args+(str(self.firstRun)+str(i))
838	<	else:
839	<	## no first run
840	<	#self.list_of_args.append([str(i)])
574	<	args=args+str(i)
575	<	if (self.sourceSeed):
576	<	if (self.sourceSeedVtx):
577	<	## pythia + vtx random seed
578	<	#self.list_of_args.append([
579	<	#                          str(self.sourceSeed)+str(i),
580	<	#                          str(self.sourceSeedVtx)+str(i)
581	<	#                          ])
582	<	args=args+str(',')+str(self.sourceSeed)+str(i)+str(',')+str(self.sourceSeedVtx)+str(i)
835	>	## pythia first run
836	>	args.append(str(self.firstRun)+str(i))
837	>	if (self.generator in self.managedGenerators):
838	>	if (self.generator == 'comphep' and i == 0):
839	>	# COMPHEP is brain-dead and wants event #'s like 1,100,200,300
840	>	args.append('1')
841		else:
842	<	## only pythia random seed
843	<	#self.list_of_args.append([(str(self.sourceSeed)+str(i))])
586	<	args=args +str(',')+str(self.sourceSeed)+str(i)
587	<	else:
588	<	## no random seed
589	<	if str(args)=='': args=args+(str(self.firstRun)+str(i))
590	<	arguments=args.split(',')
591	<	if len(arguments)==3:self.list_of_args.append([str(arguments[0]),str(arguments[1]),str(arguments[2])])
592	<	elif len(arguments)==2:self.list_of_args.append([str(arguments[0]),str(arguments[1])])
593	<	else :self.list_of_args.append([str(arguments[0])])
594	<
595	<	#   print self.list_of_args
596	<
842	>	args.append(str(i*self.eventsPerJob))
843	>	self.list_of_args.append(args)
844		return
845
846
847	<	def jobSplittingForScript(self):#CarlosDaniele
847	>	def jobSplittingForScript(self):
848		"""
849		Perform job splitting based on number of job
850		"""
#	Line 613 | Line 860 | class Cmssw(JobType):
860		# argument is seed number.$i
861		self.list_of_args = []
862		for i in range(self.total_number_of_jobs):
616	–	## Since there is no input, any site is good
617	–	# self.jobDestination.append(["Any"])
863		self.jobDestination.append([""])
619	–	## no random seed
864		self.list_of_args.append([str(i)])
865		return
866
867	<	def split(self, jobParams):
868	<
625	<	common.jobDB.load()
626	<	#### Fabio
867	>	def split(self, jobParams,firstJobID):
868	>
869		njobs = self.total_number_of_jobs
870		arglist = self.list_of_args
871	+	if njobs==0:
872	+	raise CrabException("Ask to split "+str(njobs)+" jobs: aborting")
873	+
874		# create the empty structure
875		for i in range(njobs):
876		jobParams.append("")
632	–
633	–	for job in range(njobs):
634	–	jobParams[job] = arglist[job]
635	–	# print str(arglist[job])
636	–	# print jobParams[job]
637	–	common.jobDB.setArguments(job, jobParams[job])
638	–	common.logger.debug(5,"Job "+str(job)+" Destination: "+str(self.jobDestination[job]))
639	–	common.jobDB.setDestination(job, self.jobDestination[job])
877
878	<	common.jobDB.save()
878	>	listID=[]
879	>	listField=[]
880	>	for id in range(njobs):
881	>	job = id + int(firstJobID)
882	>	jobParams[id] = arglist[id]
883	>	listID.append(job+1)
884	>	job_ToSave ={}
885	>	concString = ' '
886	>	argu=''
887	>	if len(jobParams[id]):
888	>	argu +=   concString.join(jobParams[id] )
889	>	job_ToSave['arguments']= str(job+1)+' '+argu
890	>	job_ToSave['dlsDestination']= self.jobDestination[id]
891	>	listField.append(job_ToSave)
892	>	msg="Job "+str(job)+" Arguments:   "+str(job+1)+" "+argu+"\n"  \
893	>	+"                     Destination: "+str(self.jobDestination[id])
894	>	common.logger.debug(5,msg)
895	>	common._db.updateJob_(listID,listField)
896	>	self.argsList = (len(jobParams[0])+1)
897	>
898		return
899	<
644	<	def getJobTypeArguments(self, nj, sched):
645	<	result = ''
646	<	for i in common.jobDB.arguments(nj):
647	<	result=result+str(i)+" "
648	<	return result
649	<
899	>
900		def numberOfJobs(self):
651	–	# Fabio
901		return self.total_number_of_jobs
902
903		def getTarBall(self, exe):
904		"""
905		Return the TarBall with lib and exe
906		"""
907	<
659	<	# if it exist, just return it
660	<	#
661	<	# Marco. Let's start to use relative path for Boss XML files
662	<	#
663	<	self.tgzNameWithPath = common.work_space.pathForTgz()+'share/'+self.tgz_name
907	>	self.tgzNameWithPath = common.work_space.pathForTgz()+self.tgz_name
908		if os.path.exists(self.tgzNameWithPath):
909		return self.tgzNameWithPath
910
#	Line 673 | Line 917 | class Cmssw(JobType):
917
918		# First of all declare the user Scram area
919		swArea = self.scram.getSWArea_()
676	–	#print "swArea = ", swArea
677	–	# swVersion = self.scram.getSWVersion()
678	–	# print "swVersion = ", swVersion
920		swReleaseTop = self.scram.getReleaseTop_()
921	<	#print "swReleaseTop = ", swReleaseTop
681	<
921	>
922		## check if working area is release top
923		if swReleaseTop == '' or swArea == swReleaseTop:
924	+	common.logger.debug(3,"swArea = "+swArea+" swReleaseTop ="+swReleaseTop)
925		return
926
927		import tarfile
928		try: # create tar ball
929		tar = tarfile.open(self.tgzNameWithPath, "w:gz")
930		## First find the executable
931	<	if (executable != ''):
931	>	if (self.executable != ''):
932		exeWithPath = self.scram.findFile_(executable)
933		if ( not exeWithPath ):
934		raise CrabException('User executable '+executable+' not found')
935	<
935	>
936		## then check if it's private or not
937		if exeWithPath.find(swReleaseTop) == -1:
938		# the exe is private, so we must ship
939		common.logger.debug(5,"Exe "+exeWithPath+" to be tarred")
940		path = swArea+'/'
941	<	exe = string.replace(exeWithPath, path,'')
942	<	tar.add(path+exe,executable)
941	>	# distinguish case when script is in user project area or given by full path somewhere else
942	>	if exeWithPath.find(path) >= 0 :
943	>	exe = string.replace(exeWithPath, path,'')
944	>	tar.add(path+exe,exe)
945	>	else :
946	>	tar.add(exeWithPath,os.path.basename(executable))
947		pass
948		else:
949		# the exe is from release, we'll find it on WN
950		pass
951	<
951	>
952		## Now get the libraries: only those in local working area
953	+	tar.dereference=True
954		libDir = 'lib'
955		lib = swArea+'/' +libDir
956		common.logger.debug(5,"lib "+lib+" to be tarred")
957		if os.path.exists(lib):
958		tar.add(lib,libDir)
959	<
959	>
960		## Now check if module dir is present
961		moduleDir = 'module'
962		module = swArea + '/' + moduleDir
963		if os.path.isdir(module):
964		tar.add(module,moduleDir)
965	+	tar.dereference=False
966
967		## Now check if any data dir(s) is present
968	<	swAreaLen=len(swArea)
969	<	for root, dirs, files in os.walk(swArea):
970	<	if "data" in dirs:
971	<	common.logger.debug(5,"data "+root+"/data"+" to be tarred")
972	<	tar.add(root+"/data",root[swAreaLen:]+"/data")
973	<
974	<	## Add ProdAgent dir to tar
975	<	paDir = 'ProdAgentApi'
976	<	pa = os.environ['CRABDIR'] + '/' + 'ProdAgentApi'
977	<	if os.path.isdir(pa):
978	<	tar.add(pa,paDir)
979	<
980	<	common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
968	>	self.dataExist = False
969	>	todo_list = [(i, i) for i in  os.listdir(swArea+"/src")]
970	>	while len(todo_list):
971	>	entry, name = todo_list.pop()
972	>	if name.startswith('crab_0_') or  name.startswith('.') or name == 'CVS':
973	>	continue
974	>	if os.path.isdir(swArea+"/src/"+entry):
975	>	entryPath = entry + '/'
976	>	todo_list += [(entryPath + i, i) for i in  os.listdir(swArea+"/src/"+entry)]
977	>	if name == 'data':
978	>	self.dataExist=True
979	>	common.logger.debug(5,"data "+entry+" to be tarred")
980	>	tar.add(swArea+"/src/"+entry,"src/"+entry)
981	>	pass
982	>	pass
983	>
984	>	### CMSSW ParameterSet
985	>	if not self.pset is None:
986	>	cfg_file = common.work_space.jobDir()+self.configFilename()
987	>	tar.add(cfg_file,self.configFilename())
988	>
989	>
990	>	## Add ProdCommon dir to tar
991	>	prodcommonDir = './'
992	>	prodcommonPath = os.environ['CRABDIR'] + '/' + 'external/'
993	>	neededStuff = ['ProdCommon/__init__.py','ProdCommon/FwkJobRep', 'ProdCommon/CMSConfigTools', \
994	>	'ProdCommon/Core', 'ProdCommon/MCPayloads', 'IMProv', 'ProdCommon/Storage']
995	>	for file in neededStuff:
996	>	tar.add(prodcommonPath+file,prodcommonDir+file)
997	>
998	>	##### ML stuff
999	>	ML_file_list=['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py']
1000	>	path=os.environ['CRABDIR'] + '/python/'
1001	>	for file in ML_file_list:
1002	>	tar.add(path+file,file)
1003	>
1004	>	##### Utils
1005	>	Utils_file_list=['parseCrabFjr.py','writeCfg.py', 'fillCrabFjr.py','cmscp.py']
1006	>	for file in Utils_file_list:
1007	>	tar.add(path+file,file)
1008	>
1009	>	##### AdditionalFiles
1010	>	tar.dereference=True
1011	>	for file in self.additional_inbox_files:
1012	>	tar.add(file,string.split(file,'/')[-1])
1013	>	tar.dereference=False
1014	>	common.logger.debug(5,"Files in "+self.tgzNameWithPath+" : "+str(tar.getnames()))
1015	>
1016		tar.close()
1017	<	except :
1018	<	raise CrabException('Could not create tar-ball')
1017	>	except IOError, exc:
1018	>	common.logger.write(str(exc))
1019	>	raise CrabException('Could not create tar-ball '+self.tgzNameWithPath)
1020	>	except tarfile.TarError, exc:
1021	>	common.logger.write(str(exc))
1022	>	raise CrabException('Could not create tar-ball '+self.tgzNameWithPath)
1023
1024		## check for tarball size
1025		tarballinfo = os.stat(self.tgzNameWithPath)
1026		if ( tarballinfo.st_size > self.MaxTarBallSize*1024*1024 ) :
1027	<	raise CrabException('Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' + str(self.MaxTarBallSize) + ' MB input sandbox limit and not supported by the used GRID submission system. Please make sure that no unnecessary files are in all data directories in your local CMSSW project area as they are automatically packed into the input sandbox.')
1027	>	msg  = 'Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' + str(self.MaxTarBallSize) \
1028	>	+'MB input sandbox limit \n'
1029	>	msg += '      and not supported by the direct GRID submission system.\n'
1030	>	msg += '      Please use the CRAB server mode by setting server_name=<NAME> in section [CRAB] of your crab.cfg.\n'
1031	>	msg += '      For further infos please see https://twiki.cern.ch/twiki/bin/view/CMS/CrabServer#CRABSERVER_for_Users'
1032	>	raise CrabException(msg)
1033
1034		## create tar-ball with ML stuff
1035	<	self.MLtgzfile =  common.work_space.pathForTgz()+'share/MLfiles.tgz'
1036	<	try:
746	<	tar = tarfile.open(self.MLtgzfile, "w:gz")
747	<	path=os.environ['CRABDIR'] + '/python/'
748	<	for file in ['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py', 'parseCrabFjr.py']:
749	<	tar.add(path+file,file)
750	<	common.logger.debug(5,"Files added to "+self.MLtgzfile+" : "+str(tar.getnames()))
751	<	tar.close()
752	<	except :
753	<	raise CrabException('Could not create ML files tar-ball')
754	<
755	<	return
756	<
757	<	def wsSetupEnvironment(self, nj):
1035	>
1036	>	def wsSetupEnvironment(self, nj=0):
1037		"""
1038		Returns part of a job script which prepares
1039		the execution environment for the job 'nj'.
1040		"""
1041	+	if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
1042	+	psetName = 'pset.py'
1043	+	else:
1044	+	psetName = 'pset.cfg'
1045		# Prepare JobType-independent part
1046	<	txt = ''
1047	<
1048	<	## OLI_Daniele at this level  middleware already known
766	<
767	<	txt += 'if [ $middleware == LCG ]; then \n'
1046	>	txt = '\n#Written by cms_cmssw::wsSetupEnvironment\n'
1047	>	txt += 'echo ">>> setup environment"\n'
1048	>	txt += 'if [ $middleware == LCG ]; then \n'
1049		txt += self.wsSetupCMSLCGEnvironment_()
1050		txt += 'elif [ $middleware == OSG ]; then\n'
1051		txt += '    WORKING_DIR=`/bin/mktemp  -d $OSG_WN_TMP/cms_XXXXXXXXXXXX`\n'
1052	<	txt += '    echo "Created working directory: $WORKING_DIR"\n'
1053	<	txt += '    if [ ! -d $WORKING_DIR ] ;then\n'
1054	<	txt += '        echo "SET_CMS_ENV 10016 ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
1055	<	txt += '        echo "JOB_EXIT_STATUS = 10016"\n'
775	<	txt += '        echo "JobExitCode=10016" | tee -a $RUNTIME_AREA/$repo\n'
776	<	txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
777	<	txt += '        rm -f $RUNTIME_AREA/$repo \n'
778	<	txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
779	<	txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
780	<	txt += '        exit 1\n'
1052	>	txt += '    if [ ! $? == 0 ] ;then\n'
1053	>	txt += '        echo "ERROR ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
1054	>	txt += '        job_exit_code=10016\n'
1055	>	txt += '        func_exit\n'
1056		txt += '    fi\n'
1057	+	txt += '    echo ">>> Created working directory: $WORKING_DIR"\n'
1058		txt += '\n'
1059		txt += '    echo "Change to working directory: $WORKING_DIR"\n'
1060		txt += '    cd $WORKING_DIR\n'
1061	<	txt += self.wsSetupCMSOSGEnvironment_()
1061	>	txt += '    echo ">>> current directory (WORKING_DIR): $WORKING_DIR"\n'
1062	>	txt += self.wsSetupCMSOSGEnvironment_()
1063		txt += 'fi\n'
1064
1065		# Prepare JobType-specific part
1066		scram = self.scram.commandName()
1067		txt += '\n\n'
1068	<	txt += 'echo "### SPECIFIC JOB SETUP ENVIRONMENT ###"\n'
1068	>	txt += 'echo ">>> specific cmssw setup environment:"\n'
1069	>	txt += 'echo "CMSSW_VERSION =  '+self.version+'"\n'
1070		txt += scram+' project CMSSW '+self.version+'\n'
1071		txt += 'status=$?\n'
1072		txt += 'if [ $status != 0 ] ; then\n'
1073	<	txt += '   echo "SET_EXE_ENV 10034 ==>ERROR CMSSW '+self.version+' not found on `hostname`" \n'
1074	<	txt += '   echo "JOB_EXIT_STATUS = 10034"\n'
1075	<	txt += '   echo "JobExitCode=10034" | tee -a $RUNTIME_AREA/$repo\n'
798	<	txt += '   dumpStatus $RUNTIME_AREA/$repo\n'
799	<	txt += '   rm -f $RUNTIME_AREA/$repo \n'
800	<	txt += '   echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
801	<	txt += '   echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
802	<	## OLI_Daniele
803	<	txt += '    if [ $middleware == OSG ]; then \n'
804	<	txt += '        echo "Remove working directory: $WORKING_DIR"\n'
805	<	txt += '        cd $RUNTIME_AREA\n'
806	<	txt += '        /bin/rm -rf $WORKING_DIR\n'
807	<	txt += '        if [ -d $WORKING_DIR ] ;then\n'
808	<	txt += '            echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
809	<	txt += '            echo "JOB_EXIT_STATUS = 10018"\n'
810	<	txt += '            echo "JobExitCode=10018" | tee -a $RUNTIME_AREA/$repo\n'
811	<	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
812	<	txt += '            rm -f $RUNTIME_AREA/$repo \n'
813	<	txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
814	<	txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
815	<	txt += '        fi\n'
816	<	txt += '    fi \n'
817	<	txt += '   exit 1 \n'
1073	>	txt += '    echo "ERROR ==> CMSSW '+self.version+' not found on `hostname`" \n'
1074	>	txt += '    job_exit_code=10034\n'
1075	>	txt += '    func_exit\n'
1076		txt += 'fi \n'
819	–	txt += 'echo "CMSSW_VERSION =  '+self.version+'"\n'
820	–	txt += 'export SCRAM_ARCH='+self.executable_arch+'\n'
1077		txt += 'cd '+self.version+'\n'
1078	<	### needed grep for bug in scramv1 ###
1079	<	txt += scram+' runtime -sh\n'
1078	>	txt += 'SOFTWARE_DIR=`pwd`\n'
1079	>	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
1080		txt += 'eval `'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME`\n'
1081	<	txt += 'echo $PATH\n'
1082	<
1081	>	txt += 'if [ $? != 0 ] ; then\n'
1082	>	txt += '    echo "ERROR ==> Problem with the command: "\n'
1083	>	txt += '    echo "eval \`'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME \` at `hostname`"\n'
1084	>	txt += '    job_exit_code=10034\n'
1085	>	txt += '    func_exit\n'
1086	>	txt += 'fi \n'
1087		# Handle the arguments:
1088		txt += "\n"
1089		txt += "## number of arguments (first argument always jobnumber)\n"
1090		txt += "\n"
1091	<	#        txt += "narg=$#\n"
832	<	txt += "if [ $nargs -lt 2 ]\n"
1091	>	txt += "if [ $nargs -lt "+str(self.argsList)+" ]\n"
1092		txt += "then\n"
1093	<	txt += "    echo 'SET_EXE_ENV 1 ==> ERROR Too few arguments' +$nargs+ \n"
1094	<	txt += '    echo "JOB_EXIT_STATUS = 50113"\n'
1095	<	txt += '    echo "JobExitCode=50113" | tee -a $RUNTIME_AREA/$repo\n'
837	<	txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
838	<	txt += '    rm -f $RUNTIME_AREA/$repo \n'
839	<	txt += '    echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
840	<	txt += '    echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
841	<	## OLI_Daniele
842	<	txt += '    if [ $middleware == OSG ]; then \n'
843	<	txt += '        echo "Remove working directory: $WORKING_DIR"\n'
844	<	txt += '        cd $RUNTIME_AREA\n'
845	<	txt += '        /bin/rm -rf $WORKING_DIR\n'
846	<	txt += '        if [ -d $WORKING_DIR ] ;then\n'
847	<	txt += '            echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
848	<	txt += '            echo "JOB_EXIT_STATUS = 50114"\n'
849	<	txt += '            echo "JobExitCode=50114" | tee -a $RUNTIME_AREA/$repo\n'
850	<	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
851	<	txt += '            rm -f $RUNTIME_AREA/$repo \n'
852	<	txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
853	<	txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
854	<	txt += '        fi\n'
855	<	txt += '    fi \n'
856	<	txt += "    exit 1\n"
1093	>	txt += "    echo 'ERROR ==> Too few arguments' +$nargs+ \n"
1094	>	txt += '    job_exit_code=50113\n'
1095	>	txt += "    func_exit\n"
1096		txt += "fi\n"
1097		txt += "\n"
1098
1099		# Prepare job-specific part
1100		job = common.job_list[nj]
1101	<	if self.pset != None: #CarlosDaniele
1101	>	if (self.datasetPath):
1102	>	self.primaryDataset = self.datasetPath.split("/")[1]
1103	>	DataTier = self.datasetPath.split("/")[2]
1104	>	txt += '\n'
1105	>	txt += 'DatasetPath='+self.datasetPath+'\n'
1106	>
1107	>	txt += 'PrimaryDataset='+self.primaryDataset +'\n'
1108	>	txt += 'DataTier='+DataTier+'\n'
1109	>	txt += 'ApplicationFamily=cmsRun\n'
1110	>
1111	>	else:
1112	>	self.primaryDataset = 'null'
1113	>	txt += 'DatasetPath=MCDataTier\n'
1114	>	txt += 'PrimaryDataset=null\n'
1115	>	txt += 'DataTier=null\n'
1116	>	txt += 'ApplicationFamily=MCDataTier\n'
1117	>	if self.pset != None:
1118		pset = os.path.basename(job.configFilename())
1119		txt += '\n'
1120	+	txt += 'cp  $RUNTIME_AREA/'+pset+' .\n'
1121		if (self.datasetPath): # standard job
1122	<	#txt += 'InputFiles=$2\n'
1123	<	txt += 'InputFiles=${args[1]}\n'
1124	<	txt += 'MaxEvents=${args[2]}\n'
1125	<	txt += 'SkipEvents=${args[3]}\n'
1122	>	txt += 'InputFiles=${args[1]}; export InputFiles\n'
1123	>	if (self.useParent):
1124	>	txt += 'ParentFiles=${args[2]}; export ParentFiles\n'
1125	>	txt += 'MaxEvents=${args[3]}; export MaxEvents\n'
1126	>	txt += 'SkipEvents=${args[4]}; export SkipEvents\n'
1127	>	else:
1128	>	txt += 'MaxEvents=${args[2]}; export MaxEvents\n'
1129	>	txt += 'SkipEvents=${args[3]}; export SkipEvents\n'
1130		txt += 'echo "Inputfiles:<$InputFiles>"\n'
1131	<	txt += 'sed "s#{\'INPUT\'}#$InputFiles#" $RUNTIME_AREA/'+pset+' > pset_tmp_1.cfg\n'
1131	>	if (self.useParent): txt += 'echo "ParentFiles:<$ParentFiles>"\n'
1132		txt += 'echo "MaxEvents:<$MaxEvents>"\n'
873	–	txt += 'sed "s#INPUTMAXEVENTS#$MaxEvents#" pset_tmp_1.cfg > pset_tmp_2.cfg\n'
1133		txt += 'echo "SkipEvents:<$SkipEvents>"\n'
875	–	txt += 'sed "s#INPUTSKIPEVENTS#$SkipEvents#" pset_tmp_2.cfg > pset.cfg\n'
1134		else:  # pythia like job
1135	<	if (self.sourceSeed):
1136	<	txt += 'FirstRun=${args[1]}\n'
1135	>	argNum = 1
1136	>	txt += 'PreserveSeeds='  + ','.join(self.preserveSeeds)  + '; export PreserveSeeds\n'
1137	>	txt += 'IncrementSeeds=' + ','.join(self.incrementSeeds) + '; export IncrementSeeds\n'
1138	>	txt += 'echo "PreserveSeeds: <$PreserveSeeds>"\n'
1139	>	txt += 'echo "IncrementSeeds:<$IncrementSeeds>"\n'
1140	>	if (self.firstRun):
1141	>	txt += 'export FirstRun=${args[%s]}\n' % argNum
1142		txt += 'echo "FirstRun: <$FirstRun>"\n'
1143	<	txt += 'sed "s#\<INPUTFIRSTRUN\>#$FirstRun#" $RUNTIME_AREA/'+pset+' > tmp_1.cfg\n'
1144	<	else:
1145	<	txt += '# Copy untouched pset\n'
1146	<	txt += 'cp $RUNTIME_AREA/'+pset+' tmp_1.cfg\n'
1147	<	if (self.sourceSeed):
1148	<	#                    txt += 'Seed=$2\n'
1149	<	txt += 'Seed=${args[2]}\n'
1150	<	txt += 'echo "Seed: <$Seed>"\n'
1151	<	txt += 'sed "s#\<INPUT\>#$Seed#" tmp_1.cfg > tmp_2.cfg\n'
889	<	if (self.sourceSeedVtx):
890	<	#                        txt += 'VtxSeed=$3\n'
891	<	txt += 'VtxSeed=${args[3]}\n'
892	<	txt += 'echo "VtxSeed: <$VtxSeed>"\n'
893	<	txt += 'sed "s#INPUTVTX#$VtxSeed#" tmp_2.cfg > pset.cfg\n'
894	<	else:
895	<	txt += 'mv tmp_2.cfg pset.cfg\n'
896	<	else:
897	<	txt += 'mv tmp_1.cfg pset.cfg\n'
898	<	# txt += '# Copy untouched pset\n'
899	<	# txt += 'cp $RUNTIME_AREA/'+pset+' pset.cfg\n'
1143	>	argNum += 1
1144	>	if (self.generator == 'madgraph'):
1145	>	txt += 'export FirstEvent=${args[%s]}\n' % argNum
1146	>	txt += 'echo "FirstEvent:<$FirstEvent>"\n'
1147	>	argNum += 1
1148	>	elif (self.generator == 'comphep'):
1149	>	txt += 'export CompHEPFirstEvent=${args[%s]}\n' % argNum
1150	>	txt += 'echo "CompHEPFirstEvent:<$CompHEPFirstEvent>"\n'
1151	>	argNum += 1
1152
1153	+	txt += 'mv -f ' + pset + ' ' + psetName + '\n'
1154
1155	<	if len(self.additional_inbox_files) > 0:
1156	<	for file in self.additional_inbox_files:
1157	<	relFile = file.split("/")[-1]
905	<	txt += 'if [ -e $RUNTIME_AREA/'+relFile+' ] ; then\n'
906	<	txt += '   cp $RUNTIME_AREA/'+relFile+' .\n'
907	<	txt += '   chmod +x '+relFile+'\n'
908	<	txt += 'fi\n'
909	<	pass
910	<
911	<	if self.pset != None: #CarlosDaniele
912	<	txt += 'echo "### END JOB SETUP ENVIRONMENT ###"\n\n'
913	<
1155	>
1156	>	if self.pset != None:
1157	>	# FUTURE: Can simply for 2_1_x and higher
1158		txt += '\n'
1159	<	txt += 'echo "***** cat pset.cfg *********"\n'
1160	<	txt += 'cat pset.cfg\n'
1161	<	txt += 'echo "****** end pset.cfg ********"\n'
1159	>	if self.debug_wrapper==True:
1160	>	txt += 'echo "***** cat ' + psetName + ' *********"\n'
1161	>	txt += 'cat ' + psetName + '\n'
1162	>	txt += 'echo "****** end ' + psetName + ' ********"\n'
1163	>	txt += '\n'
1164	>	if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
1165	>	txt += 'PSETHASH=`edmConfigHash ' + psetName + '` \n'
1166	>	else:
1167	>	txt += 'PSETHASH=`edmConfigHash < ' + psetName + '` \n'
1168	>	txt += 'echo "PSETHASH = $PSETHASH" \n'
1169		txt += '\n'
919	–	# txt += 'echo "***** cat pset1.cfg *********"\n'
920	–	# txt += 'cat pset1.cfg\n'
921	–	# txt += 'echo "****** end pset1.cfg ********"\n'
1170		return txt
1171
1172	<	def wsBuildExe(self, nj=0):
1172	>	def wsUntarSoftware(self, nj=0):
1173		"""
1174		Put in the script the commands to build an executable
1175		or a library.
1176		"""
1177
1178	<	txt = ""
1178	>	txt = '\n#Written by cms_cmssw::wsUntarSoftware\n'
1179
1180		if os.path.isfile(self.tgzNameWithPath):
1181	<	txt += 'echo "tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'"\n'
1182	<	txt += 'tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
1181	>	txt += 'echo ">>> tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+' :" \n'
1182	>	txt += 'tar xzf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
1183	>	if  self.debug_wrapper:
1184	>	txt += 'tar tzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
1185	>	txt += 'ls -Al \n'
1186		txt += 'untar_status=$? \n'
1187		txt += 'if [ $untar_status -ne 0 ]; then \n'
1188	<	txt += '   echo "SET_EXE 1 ==> ERROR Untarring .tgz file failed"\n'
1189	<	txt += '   echo "JOB_EXIT_STATUS = $untar_status" \n'
1190	<	txt += '   echo "JobExitCode=$untar_status" | tee -a $RUNTIME_AREA/$repo\n'
940	<	txt += '   if [ $middleware == OSG ]; then \n'
941	<	txt += '       echo "Remove working directory: $WORKING_DIR"\n'
942	<	txt += '       cd $RUNTIME_AREA\n'
943	<	txt += '       /bin/rm -rf $WORKING_DIR\n'
944	<	txt += '       if [ -d $WORKING_DIR ] ;then\n'
945	<	txt += '           echo "SET_EXE 50999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Untarring .tgz file failed"\n'
946	<	txt += '           echo "JOB_EXIT_STATUS = 50999"\n'
947	<	txt += '           echo "JobExitCode=50999" | tee -a $RUNTIME_AREA/$repo\n'
948	<	txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
949	<	txt += '           rm -f $RUNTIME_AREA/$repo \n'
950	<	txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
951	<	txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
952	<	txt += '       fi\n'
953	<	txt += '   fi \n'
954	<	txt += '   \n'
955	<	txt += '   exit 1 \n'
1188	>	txt += '   echo "ERROR ==> Untarring .tgz file failed"\n'
1189	>	txt += '   job_exit_code=$untar_status\n'
1190	>	txt += '   func_exit\n'
1191		txt += 'else \n'
1192		txt += '   echo "Successful untar" \n'
1193		txt += 'fi \n'
1194		txt += '\n'
1195	<	txt += 'echo "Include ProdAgentApi in PYTHONPATH"\n'
1195	>	txt += 'echo ">>> Include $RUNTIME_AREA in PYTHONPATH:"\n'
1196		txt += 'if [ -z "$PYTHONPATH" ]; then\n'
1197	<	txt += '   export PYTHONPATH=ProdAgentApi\n'
1197	>	txt += '   export PYTHONPATH=$RUNTIME_AREA/\n'
1198		txt += 'else\n'
1199	<	txt += '   export PYTHONPATH=ProdAgentApi:${PYTHONPATH}\n'
1199	>	txt += '   export PYTHONPATH=$RUNTIME_AREA/:${PYTHONPATH}\n'
1200	>	txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
1201		txt += 'fi\n'
1202		txt += '\n'
1203
1204		pass
1205	<
1205	>
1206		return txt
1207
1208	<	def modifySteeringCards(self, nj):
1208	>	def wsBuildExe(self, nj=0):
1209		"""
1210	<	modify the card provided by the user,
1211	<	writing a new card into share dir
1210	>	Put in the script the commands to build an executable
1211	>	or a library.
1212		"""
1213	<
1213	>
1214	>	txt = '\n#Written by cms_cmssw::wsBuildExe\n'
1215	>	txt += 'echo ">>> moving CMSSW software directories in `pwd`" \n'
1216	>
1217	>	txt += 'rm -r lib/ module/ \n'
1218	>	txt += 'mv $RUNTIME_AREA/lib/ . \n'
1219	>	txt += 'mv $RUNTIME_AREA/module/ . \n'
1220	>	if self.dataExist == True:
1221	>	txt += 'rm -r src/ \n'
1222	>	txt += 'mv $RUNTIME_AREA/src/ . \n'
1223	>	if len(self.additional_inbox_files)>0:
1224	>	for file in self.additional_inbox_files:
1225	>	txt += 'mv $RUNTIME_AREA/'+os.path.basename(file)+' . \n'
1226	>	# txt += 'mv $RUNTIME_AREA/ProdCommon/ . \n'
1227	>	# txt += 'mv $RUNTIME_AREA/IMProv/ . \n'
1228	>
1229	>	txt += 'echo ">>> Include $RUNTIME_AREA in PYTHONPATH:"\n'
1230	>	txt += 'if [ -z "$PYTHONPATH" ]; then\n'
1231	>	txt += '   export PYTHONPATH=$RUNTIME_AREA/\n'
1232	>	txt += 'else\n'
1233	>	txt += '   export PYTHONPATH=$RUNTIME_AREA/:${PYTHONPATH}\n'
1234	>	txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
1235	>	txt += 'fi\n'
1236	>	txt += '\n'
1237	>
1238	>	return txt
1239	>
1240	>
1241		def executableName(self):
1242	<	if self.scriptExe: #CarlosDaniele
1242	>	if self.scriptExe:
1243		return "sh "
1244		else:
1245		return self.executable
1246
1247		def executableArgs(self):
1248	+	# FUTURE: This function tests the CMSSW version. Can be simplified as we drop support for old versions
1249		if self.scriptExe:#CarlosDaniele
1250		return   self.scriptExe + " $NJob"
1251	<	else:
1252	<	return " -p pset.cfg"
1251	>	else:
1252	>	ex_args = ""
1253	>	# FUTURE: This tests the CMSSW version. Can remove code as versions deprecated
1254	>	# Framework job report
1255	>	if (self.CMSSW_major >= 1 and self.CMSSW_minor >= 5) or (self.CMSSW_major >= 2):
1256	>	ex_args += " -j $RUNTIME_AREA/crab_fjr_$NJob.xml"
1257	>	# Type of config file
1258	>	if self.CMSSW_major >= 2 :
1259	>	ex_args += " -p pset.py"
1260	>	else:
1261	>	ex_args += " -p pset.cfg"
1262	>	return ex_args
1263
1264		def inputSandbox(self, nj):
1265		"""
1266		Returns a list of filenames to be put in JDL input sandbox.
1267		"""
1268		inp_box = []
995	–	# # dict added to delete duplicate from input sandbox file list
996	–	# seen = {}
997	–	## code
1269		if os.path.isfile(self.tgzNameWithPath):
1270		inp_box.append(self.tgzNameWithPath)
1271	<	if os.path.isfile(self.MLtgzfile):
1001	<	inp_box.append(self.MLtgzfile)
1002	<	## config
1003	<	if not self.pset is None:
1004	<	inp_box.append(common.work_space.pathForTgz() + 'job/' + self.configFilename())
1005	<	## additional input files
1006	<	for file in self.additional_inbox_files:
1007	<	inp_box.append(file)
1271	>	inp_box.append(common.work_space.jobDir() + self.scriptName)
1272		return inp_box
1273
1274		def outputSandbox(self, nj):
#	Line 1015 | Line 1279 | class Cmssw(JobType):
1279
1280		## User Declared output files
1281		for out in (self.output_file+self.output_file_sandbox):
1282	<	n_out = nj + 1
1283	<	out_box.append(self.numberFile_(out,str(n_out)))
1282	>	n_out = nj + 1
1283	>	out_box.append(numberFile(out,str(n_out)))
1284		return out_box
1285
1022	–	def prepareSteeringCards(self):
1023	–	"""
1024	–	Make initial modifications of the user's steering card file.
1025	–	"""
1026	–	return
1286
1287		def wsRenameOutput(self, nj):
1288		"""
1289		Returns part of a job script which renames the produced files.
1290		"""
1291
1292	<	txt = '\n'
1293	<	txt += '# directory content\n'
1294	<	txt += 'ls \n'
1292	>	txt = '\n#Written by cms_cmssw::wsRenameOutput\n'
1293	>	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
1294	>	txt += 'echo ">>> current directory content:"\n'
1295	>	if self.debug_wrapper:
1296	>	txt += 'ls -Al\n'
1297	>	txt += '\n'
1298
1299	<	for fileWithSuffix in (self.output_file+self.output_file_sandbox):
1300	<	output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1299	>	for fileWithSuffix in (self.output_file):
1300	>	output_file_num = numberFile(fileWithSuffix, '$NJob')
1301		txt += '\n'
1302		txt += '# check output file\n'
1303	<	txt += 'ls '+fileWithSuffix+'\n'
1304	<	txt += 'ls_result=$?\n'
1305	<	txt += 'if [ $ls_result -ne 0 ] ; then\n'
1306	<	txt += '   echo "ERROR: Problem with output file"\n'
1307	<	if common.scheduler.boss_scheduler_name == 'condor_g':
1303	>	txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1304	>	if (self.copy_data == 1):  # For OSG nodes, file is in $WORKING_DIR, should not be moved to $RUNTIME_AREA
1305	>	txt += '    mv '+fileWithSuffix+' '+output_file_num+'\n'
1306	>	txt += '    ln -s `pwd`/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1307	>	else:
1308	>	txt += '    mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1309	>	txt += '    ln -s $RUNTIME_AREA/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1310	>	txt += 'else\n'
1311	>	txt += '    job_exit_code=60302\n'
1312	>	txt += '    echo "WARNING: Output file '+fileWithSuffix+' not found"\n'
1313	>	if common.scheduler.name().upper() == 'CONDOR_G':
1314		txt += '    if [ $middleware == OSG ]; then \n'
1315		txt += '        echo "prepare dummy output file"\n'
1316		txt += '        echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
1317		txt += '    fi \n'
1050	–	txt += 'else\n'
1051	–	txt += '   cp '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1318		txt += 'fi\n'
1319	<
1320	<	txt += 'cd $RUNTIME_AREA\n'
1321	<	txt += 'cd $RUNTIME_AREA\n'
1056	<	### OLI_DANIELE
1057	<	txt += 'if [ $middleware == OSG ]; then\n'
1058	<	txt += '    cd $RUNTIME_AREA\n'
1059	<	txt += '    echo "Remove working directory: $WORKING_DIR"\n'
1060	<	txt += '    /bin/rm -rf $WORKING_DIR\n'
1061	<	txt += '    if [ -d $WORKING_DIR ] ;then\n'
1062	<	txt += '        echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1063	<	txt += '        echo "JOB_EXIT_STATUS = 60999"\n'
1064	<	txt += '        echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
1065	<	txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
1066	<	txt += '        rm -f $RUNTIME_AREA/$repo \n'
1067	<	txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1068	<	txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1069	<	txt += '    fi\n'
1070	<	txt += 'fi\n'
1071	<	txt += '\n'
1072	<
1073	<	file_list = ''
1074	<	## Add to filelist only files to be possibly copied to SE
1075	<	for fileWithSuffix in self.output_file:
1076	<	output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1077	<	file_list=file_list+output_file_num+' '
1078	<	file_list=file_list[:-1]
1079	<	txt += 'file_list="'+file_list+'"\n'
1319	>	file_list = []
1320	>	for fileWithSuffix in (self.output_file):
1321	>	file_list.append(numberFile('$SOFTWARE_DIR/'+fileWithSuffix, '$NJob'))
1322
1323	+	txt += 'file_list="'+string.join(file_list,',')+'"\n'
1324	+	txt += '\n'
1325	+	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
1326	+	txt += 'echo ">>> current directory content:"\n'
1327	+	if self.debug_wrapper:
1328	+	txt += 'ls -Al\n'
1329	+	txt += '\n'
1330	+	txt += 'cd $RUNTIME_AREA\n'
1331	+	txt += 'echo ">>> current directory (RUNTIME_AREA):  $RUNTIME_AREA"\n'
1332		return txt
1333
1083	–	def numberFile_(self, file, txt):
1084	–	"""
1085	–	append _'txt' before last extension of a file
1086	–	"""
1087	–	p = string.split(file,".")
1088	–	# take away last extension
1089	–	name = p[0]
1090	–	for x in p[1:-1]:
1091	–	name=name+"."+x
1092	–	# add "_txt"
1093	–	if len(p)>1:
1094	–	ext = p[len(p)-1]
1095	–	result = name + '_' + txt + "." + ext
1096	–	else:
1097	–	result = name + '_' + txt
1098	–
1099	–	return result
1100	–
1334		def getRequirements(self, nj=[]):
1335		"""
1336	<	return job requirements to add to jdl files
1336	>	return job requirements to add to jdl files
1337		"""
1338		req = ''
1339		if self.version:
1340		req='Member("VO-cms-' + \
1341		self.version + \
1342		'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1343	+	if self.executable_arch:
1344	+	req+=' && Member("VO-cms-' + \
1345	+	self.executable_arch + \
1346	+	'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1347
1348		req = req + ' && (other.GlueHostNetworkAdapterOutboundIP)'
1349	+	if ( common.scheduler.name() == "glitecoll" ) or ( common.scheduler.name() == "glite"):
1350	+	req += ' && other.GlueCEStateStatus == "Production" '
1351
1352		return req
1353
1354		def configFilename(self):
1355		""" return the config filename """
1356	<	return self.name()+'.cfg'
1356	>	# FUTURE: Can remove cfg mode for CMSSW >= 2_1_x
1357	>	if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
1358	>	return self.name()+'.py'
1359	>	else:
1360	>	return self.name()+'.cfg'
1361
1119	–	### OLI_DANIELE
1362		def wsSetupCMSOSGEnvironment_(self):
1363		"""
1364		Returns part of a job script which is prepares
1365		the execution environment and which is common for all CMS jobs.
1366		"""
1367	<	txt = '\n'
1368	<	txt += '   echo "### SETUP CMS OSG  ENVIRONMENT ###"\n'
1369	<	txt += '   if [ -f $GRID3_APP_DIR/cmssoft/cmsset_default.sh ] ;then\n'
1370	<	txt += '      # Use $GRID3_APP_DIR/cmssoft/cmsset_default.sh to setup cms software\n'
1371	<	txt += '       source $GRID3_APP_DIR/cmssoft/cmsset_default.sh '+self.version+'\n'
1372	<	txt += '   elif [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n'
1367	>	txt = '\n#Written by cms_cmssw::wsSetupCMSOSGEnvironment_\n'
1368	>	txt += '    echo ">>> setup CMS OSG environment:"\n'
1369	>	txt += '    echo "set SCRAM ARCH to ' + self.executable_arch + '"\n'
1370	>	txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
1371	>	txt += '    echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
1372	>	txt += '    if [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n'
1373		txt += '      # Use $OSG_APP/cmssoft/cms/cmsset_default.sh to setup cms software\n'
1374	<	txt += '       source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1375	<	txt += '   else\n'
1376	<	txt += '       echo "SET_CMS_ENV 10020 ==> ERROR $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1377	<	txt += '       echo "JOB_EXIT_STATUS = 10020"\n'
1378	<	txt += '       echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
1379	<	txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
1138	<	txt += '       rm -f $RUNTIME_AREA/$repo \n'
1139	<	txt += '       echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1140	<	txt += '       echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1141	<	txt += '       exit 1\n'
1142	<	txt += '\n'
1143	<	txt += '       echo "Remove working directory: $WORKING_DIR"\n'
1144	<	txt += '       cd $RUNTIME_AREA\n'
1145	<	txt += '       /bin/rm -rf $WORKING_DIR\n'
1146	<	txt += '       if [ -d $WORKING_DIR ] ;then\n'
1147	<	txt += '            echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1148	<	txt += '            echo "JOB_EXIT_STATUS = 10017"\n'
1149	<	txt += '            echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n'
1150	<	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
1151	<	txt += '            rm -f $RUNTIME_AREA/$repo \n'
1152	<	txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1153	<	txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1154	<	txt += '       fi\n'
1155	<	txt += '\n'
1156	<	txt += '       exit 1\n'
1157	<	txt += '   fi\n'
1374	>	txt += '        source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1375	>	txt += '    else\n'
1376	>	txt += '        echo "ERROR ==> $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1377	>	txt += '        job_exit_code=10020\n'
1378	>	txt += '        func_exit\n'
1379	>	txt += '    fi\n'
1380		txt += '\n'
1381	<	txt += '   echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1382	<	txt += '   echo " END SETUP CMS OSG  ENVIRONMENT "\n'
1381	>	txt += '    echo "==> setup cms environment ok"\n'
1382	>	txt += '    echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
1383
1384		return txt
1385	<
1164	<	### OLI_DANIELE
1385	>
1386		def wsSetupCMSLCGEnvironment_(self):
1387		"""
1388		Returns part of a job script which is prepares
1389		the execution environment and which is common for all CMS jobs.
1390		"""
1391	<	txt  = '   \n'
1392	<	txt += '   echo " ### SETUP CMS LCG  ENVIRONMENT ### "\n'
1393	<	txt += '   if [ ! $VO_CMS_SW_DIR ] ;then\n'
1394	<	txt += '       echo "SET_CMS_ENV 10031 ==> ERROR CMS software dir not found on WN `hostname`"\n'
1395	<	txt += '       echo "JOB_EXIT_STATUS = 10031" \n'
1396	<	txt += '       echo "JobExitCode=10031" | tee -a $RUNTIME_AREA/$repo\n'
1397	<	txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
1398	<	txt += '       rm -f $RUNTIME_AREA/$repo \n'
1399	<	txt += '       echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1400	<	txt += '       echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1401	<	txt += '       exit 1\n'
1402	<	txt += '   else\n'
1403	<	txt += '       echo "Sourcing environment... "\n'
1404	<	txt += '       if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
1405	<	txt += '           echo "SET_CMS_ENV 10020 ==> ERROR cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1406	<	txt += '           echo "JOB_EXIT_STATUS = 10020"\n'
1407	<	txt += '           echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
1408	<	txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1409	<	txt += '           rm -f $RUNTIME_AREA/$repo \n'
1410	<	txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1411	<	txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1412	<	txt += '           exit 1\n'
1413	<	txt += '       fi\n'
1414	<	txt += '       echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1415	<	txt += '       source $VO_CMS_SW_DIR/cmsset_default.sh\n'
1416	<	txt += '       result=$?\n'
1417	<	txt += '       if [ $result -ne 0 ]; then\n'
1418	<	txt += '           echo "SET_CMS_ENV 10032 ==> ERROR problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1419	<	txt += '           echo "JOB_EXIT_STATUS = 10032"\n'
1420	<	txt += '           echo "JobExitCode=10032" | tee -a $RUNTIME_AREA/$repo\n'
1421	<	txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1422	<	txt += '           rm -f $RUNTIME_AREA/$repo \n'
1423	<	txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1424	<	txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1425	<	txt += '           exit 1\n'
1426	<	txt += '       fi\n'
1427	<	txt += '   fi\n'
1428	<	txt += '   \n'
1429	<	txt += '   echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1430	<	txt += '   echo "### END SETUP CMS LCG ENVIRONMENT ###"\n'
1391	>	txt = '\n#Written by cms_cmssw::wsSetupCMSLCGEnvironment_\n'
1392	>	txt += '    echo ">>> setup CMS LCG environment:"\n'
1393	>	txt += '    echo "set SCRAM ARCH and BUILD_ARCH to ' + self.executable_arch + ' ###"\n'
1394	>	txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
1395	>	txt += '    export BUILD_ARCH='+self.executable_arch+'\n'
1396	>	txt += '    if [ ! $VO_CMS_SW_DIR ] ;then\n'
1397	>	txt += '        echo "ERROR ==> CMS software dir not found on WN `hostname`"\n'
1398	>	txt += '        job_exit_code=10031\n'
1399	>	txt += '        func_exit\n'
1400	>	txt += '    else\n'
1401	>	txt += '        echo "Sourcing environment... "\n'
1402	>	txt += '        if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
1403	>	txt += '            echo "ERROR ==> cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1404	>	txt += '            job_exit_code=10020\n'
1405	>	txt += '            func_exit\n'
1406	>	txt += '        fi\n'
1407	>	txt += '        echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1408	>	txt += '        source $VO_CMS_SW_DIR/cmsset_default.sh\n'
1409	>	txt += '        result=$?\n'
1410	>	txt += '        if [ $result -ne 0 ]; then\n'
1411	>	txt += '            echo "ERROR ==> problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1412	>	txt += '            job_exit_code=10032\n'
1413	>	txt += '            func_exit\n'
1414	>	txt += '        fi\n'
1415	>	txt += '    fi\n'
1416	>	txt += '    \n'
1417	>	txt += '    echo "==> setup cms environment ok"\n'
1418	>	return txt
1419	>
1420	>	def wsModifyReport(self, nj):
1421	>	"""
1422	>	insert the part of the script that modifies the FrameworkJob Report
1423	>	"""
1424	>	txt = '\n#Written by cms_cmssw::wsModifyReport\n'
1425	>	publish_data = int(self.cfg_params.get('USER.publish_data',0))
1426	>	if (publish_data == 1):
1427	>
1428	>	processedDataset = self.cfg_params['USER.publish_data_name']
1429	>
1430	>	txt += 'if [ $StageOutExitStatus -eq 0 ]; then\n'
1431	>	txt += '    FOR_LFN=$LFNBaseName\n'
1432	>	txt += 'else\n'
1433	>	txt += '    FOR_LFN=/copy_problems/ \n'
1434	>	txt += '    SE=""\n'
1435	>	txt += '    SE_PATH=""\n'
1436	>	txt += 'fi\n'
1437	>
1438	>	txt += 'echo ">>> Modify Job Report:" \n'
1439	>	txt += 'chmod a+x $RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py\n'
1440	>	txt += 'ProcessedDataset='+processedDataset+'\n'
1441	>	#txt += 'ProcessedDataset=$procDataset \n'
1442	>	txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1443	>	txt += 'echo "SE = $SE"\n'
1444	>	txt += 'echo "SE_PATH = $SE_PATH"\n'
1445	>	txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1446	>	txt += 'echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n'
1447	>	args = '$RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier ' \
1448	>	'$USER-$ProcessedDataset-$PSETHASH $ApplicationFamily '+ \
1449	>	'  $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH'
1450	>	txt += 'echo "$RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py '+str(args)+'"\n'
1451	>	txt += '$RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py '+str(args)+'\n'
1452	>	txt += 'modifyReport_result=$?\n'
1453	>	txt += 'if [ $modifyReport_result -ne 0 ]; then\n'
1454	>	txt += '    modifyReport_result=70500\n'
1455	>	txt += '    job_exit_code=$modifyReport_result\n'
1456	>	txt += '    echo "ModifyReportResult=$modifyReport_result" | tee -a $RUNTIME_AREA/$repo\n'
1457	>	txt += '    echo "WARNING: Problem with ModifyJobReport"\n'
1458	>	txt += 'else\n'
1459	>	txt += '    mv NewFrameworkJobReport.xml $RUNTIME_AREA/crab_fjr_$NJob.xml\n'
1460	>	txt += 'fi\n'
1461	>	return txt
1462	>
1463	>	def wsParseFJR(self):
1464	>	"""
1465	>	Parse the FrameworkJobReport to obtain useful infos
1466	>	"""
1467	>	txt = '\n#Written by cms_cmssw::wsParseFJR\n'
1468	>	txt += 'echo ">>> Parse FrameworkJobReport crab_fjr.xml"\n'
1469	>	txt += 'if [ -s $RUNTIME_AREA/crab_fjr_$NJob.xml ]; then\n'
1470	>	txt += '    if [ -s $RUNTIME_AREA/parseCrabFjr.py ]; then\n'
1471	>	txt += '        cmd_out=`python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --dashboard $MonitorID,$MonitorJobID '+self.debugWrap+'`\n'
1472	>	if self.debug_wrapper :
1473	>	txt += '        echo "Result of parsing the FrameworkJobReport crab_fjr.xml: $cmd_out"\n'
1474	>	txt += '        executable_exit_status=`python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --exitcode`\n'
1475	>	txt += '        if [ $executable_exit_status -eq 50115 ];then\n'
1476	>	txt += '            echo ">>> crab_fjr.xml contents: "\n'
1477	>	txt += '            cat $RUNTIME_AREA/crab_fjr_$NJob.xml\n'
1478	>	txt += '            echo "Wrong FrameworkJobReport --> does not contain useful info. ExitStatus: $executable_exit_status"\n'
1479	>	txt += '        elif [ $executable_exit_status -eq -999 ];then\n'
1480	>	txt += '            echo "ExitStatus from FrameworkJobReport not available. not available. Using exit code of executable from command line."\n'
1481	>	txt += '        else\n'
1482	>	txt += '            echo "Extracted ExitStatus from FrameworkJobReport parsing output: $executable_exit_status"\n'
1483	>	txt += '        fi\n'
1484	>	txt += '    else\n'
1485	>	txt += '        echo "CRAB python script to parse CRAB FrameworkJobReport crab_fjr.xml is not available, using exit code of executable from command line."\n'
1486	>	txt += '    fi\n'
1487	>	#### Patch to check input data reading for CMSSW16x Hopefully we-ll remove it asap
1488	>	txt += '    if [ $executable_exit_status -eq 0 ];then\n'
1489	>	txt += '      echo ">>> Executable succeded  $executable_exit_status"\n'
1490	>	if (self.datasetPath and not (self.dataset_pu or self.useParent)) :
1491	>	# VERIFY PROCESSED DATA
1492	>	txt += '      echo ">>> Verify list of processed files:"\n'
1493	>	txt += '      echo $InputFiles |tr -d \'\\\\\' |tr \',\' \'\\n\'|tr -d \'"\' > input-files.txt\n'
1494	>	txt += '      python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --lfn > processed-files.txt\n'
1495	>	txt += '      cat input-files.txt  | sort | uniq > tmp.txt\n'
1496	>	txt += '      mv tmp.txt input-files.txt\n'
1497	>	txt += '      echo "cat input-files.txt"\n'
1498	>	txt += '      echo "----------------------"\n'
1499	>	txt += '      cat input-files.txt\n'
1500	>	txt += '      cat processed-files.txt | sort | uniq > tmp.txt\n'
1501	>	txt += '      mv tmp.txt processed-files.txt\n'
1502	>	txt += '      echo "----------------------"\n'
1503	>	txt += '      echo "cat processed-files.txt"\n'
1504	>	txt += '      echo "----------------------"\n'
1505	>	txt += '      cat processed-files.txt\n'
1506	>	txt += '      echo "----------------------"\n'
1507	>	txt += '      diff -q input-files.txt processed-files.txt\n'
1508	>	txt += '      fileverify_status=$?\n'
1509	>	txt += '      if [ $fileverify_status -ne 0 ]; then\n'
1510	>	txt += '         executable_exit_status=30001\n'
1511	>	txt += '         echo "ERROR ==> not all input files processed"\n'
1512	>	txt += '         echo "      ==> list of processed files from crab_fjr.xml differs from list in pset.cfg"\n'
1513	>	txt += '         echo "      ==> diff input-files.txt processed-files.txt"\n'
1514	>	txt += '      fi\n'
1515	>	txt += '    elif [ $executable_exit_status -ne 0 ] || [ $executable_exit_status -ne 50015 ] || [ $executable_exit_status -ne 50017 ];then\n'
1516	>	txt += '      echo ">>> Executable failed  $executable_exit_status"\n'
1517	>	txt += '      echo "ExeExitCode=$executable_exit_status" | tee -a $RUNTIME_AREA/$repo\n'
1518	>	txt += '      echo "EXECUTABLE_EXIT_STATUS = $executable_exit_status"\n'
1519	>	txt += '      job_exit_code=$executable_exit_status\n'
1520	>	txt += '      func_exit\n'
1521	>	txt += '    fi\n'
1522	>	txt += '\n'
1523	>	txt += 'else\n'
1524	>	txt += '    echo "CRAB FrameworkJobReport crab_fjr.xml is not available, using exit code of executable from command line."\n'
1525	>	txt += 'fi\n'
1526	>	txt += '\n'
1527	>	txt += 'echo "ExeExitCode=$executable_exit_status" | tee -a $RUNTIME_AREA/$repo\n'
1528	>	txt += 'echo "EXECUTABLE_EXIT_STATUS = $executable_exit_status"\n'
1529	>	txt += 'job_exit_code=$executable_exit_status\n'
1530	>
1531		return txt
1532
1533		def setParam_(self, param, value):
#	Line 1215 | Line 1536 | class Cmssw(JobType):
1536		def getParams(self):
1537		return self._params
1538
1218	–	def setTaskid_(self):
1219	–	self._taskId = self.cfg_params['taskId']
1220	–
1221	–	def getTaskid(self):
1222	–	return self._taskId
1223	–
1224	–	#######################################################################
1539		def uniquelist(self, old):
1540		"""
1541		remove duplicates from a list
#	Line 1230 | Line 1544 | class Cmssw(JobType):
1544		for e in old:
1545		nd[e]=0
1546		return nd.keys()
1547	+
1548	+	def outList(self,list=False):
1549	+	"""
1550	+	check the dimension of the output files
1551	+	"""
1552	+	txt = ''
1553	+	txt += 'echo ">>> list of expected files on output sandbox"\n'
1554	+	listOutFiles = []
1555	+	stdout = 'CMSSW_$NJob.stdout'
1556	+	stderr = 'CMSSW_$NJob.stderr'
1557	+	if (self.return_data == 1):
1558	+	for file in (self.output_file+self.output_file_sandbox):
1559	+	listOutFiles.append(numberFile(file, '$NJob'))
1560	+	listOutFiles.append(stdout)
1561	+	listOutFiles.append(stderr)
1562	+	else:
1563	+	for file in (self.output_file_sandbox):
1564	+	listOutFiles.append(numberFile(file, '$NJob'))
1565	+	listOutFiles.append(stdout)
1566	+	listOutFiles.append(stderr)
1567	+	txt += 'echo "output files: '+string.join(listOutFiles,' ')+'"\n'
1568	+	txt += 'filesToCheck="'+string.join(listOutFiles,' ')+'"\n'
1569	+	txt += 'export filesToCheck\n'
1570	+	if list : return self.output_file
1571	+	return txt

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines