[ViewVC] Diff of: cvsroot/COMP/CRAB/python/cms

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.107 by gutsche, Tue Jul 24 19:49:35 2007 UTC vs.
Revision 1.254 by ewv, Thu Oct 30 16:25:24 2008 UTC

#	Line 2 \| Line 2 \| from JobType import JobType
2		from crab_logger import Logger
3		from crab_exceptions import *
4		from crab_util import *
5	+	from WMCore.SiteScreening.BlackWhiteListParser import SEBlackWhiteListParser
6		import common
7		import Scram
8
9		import os, string, glob
10
11		class Cmssw(JobType):
12	<	def __init__(self, cfg_params, ncjobs):
12	>	def __init__(self, cfg_params, ncjobs,skip_blocks, isNew):
13		JobType.__init__(self, 'CMSSW')
14		common.logger.debug(3,'CMSSW::__init__')
15	+	self.skip_blocks = skip_blocks
16	+
17	+	self.argsList = []
18
19		self._params = {}
20		self.cfg_params = cfg_params
21
22	<	try:
23	<	self.MaxTarBallSize = float(self.cfg_params['EDG.maxtarballsize'])
24	<	except KeyError:
25	<	self.MaxTarBallSize = 9.5
22	>	# init BlackWhiteListParser
23	>	seWhiteList = cfg_params.get('EDG.se_white_list',[])
24	>	seBlackList = cfg_params.get('EDG.se_black_list',[])
25	>	self.blackWhiteListParser = SEBlackWhiteListParser(seWhiteList, seBlackList, common.logger)
26	>
27	>	### Temporary patch to automatically skip the ISB size check:
28	>	server=self.cfg_params.get('CRAB.server_name',None)
29	>	size = 9.5
30	>	if server or common.scheduler.name().upper() in ['LSF','CAF']: size = 99999
31	>	### D.S.
32	>	self.MaxTarBallSize = float(self.cfg_params.get('EDG.maxtarballsize',size))
33
34		# number of jobs requested to be created, limit obj splitting
35		self.ncjobs = ncjobs
36
37		log = common.logger
38	<
38	>
39		self.scram = Scram.Scram(cfg_params)
40		self.additional_inbox_files = []
41		self.scriptExe = ''
42		self.executable = ''
43		self.executable_arch = self.scram.getArch()
44		self.tgz_name = 'default.tgz'
34	–	self.additional_tgz_name = 'additional.tgz'
45		self.scriptName = 'CMSSW.sh'
46	<	self.pset = '' #scrip use case Da
47	<	self.datasetPath = '' #scrip use case Da
46	>	self.pset = ''
47	>	self.datasetPath = ''
48
49		# set FJR file name
50		self.fjrFileName = 'crab_fjr.xml'
51
52		self.version = self.scram.getSWVersion()
53	<	common.taskDB.setDict('codeVersion',self.version)
54	<	self.setParam_('application', self.version)
53	>	version_array = self.version.split('_')
54	>	self.CMSSW_major = 0
55	>	self.CMSSW_minor = 0
56	>	self.CMSSW_patch = 0
57	>	try:
58	>	self.CMSSW_major = int(version_array[1])
59	>	self.CMSSW_minor = int(version_array[2])
60	>	self.CMSSW_patch = int(version_array[3])
61	>	except:
62	>	msg = "Cannot parse CMSSW version string: " + self.version + " for major and minor release number!"
63	>	raise CrabException(msg)
64
65		### collect Data cards
66
67	<	## get DBS mode
68	<	try:
50	<	self.use_dbs_1 = int(self.cfg_params['CMSSW.use_dbs_1'])
51	<	except KeyError:
52	<	self.use_dbs_1 = 0
53	<
54	<	try:
55	<	tmp = cfg_params['CMSSW.datasetpath']
56	<	log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
57	<	if string.lower(tmp)=='none':
58	<	self.datasetPath = None
59	<	self.selectNoInput = 1
60	<	else:
61	<	self.datasetPath = tmp
62	<	self.selectNoInput = 0
63	<	except KeyError:
64	<	msg = "Error: datasetpath not defined "
67	>	if not cfg_params.has_key('CMSSW.datasetpath'):
68	>	msg = "Error: datasetpath not defined "
69		raise CrabException(msg)
70
71	<	# ML monitoring
72	<	# split dataset path style: /PreProdR3Minbias/SIM/GEN-SIM
73	<	if not self.datasetPath:
74	<	self.setParam_('dataset', 'None')
75	<	self.setParam_('owner', 'None')
71	>	### Temporary: added to remove input file control in the case of PU
72	>	self.dataset_pu = cfg_params.get('CMSSW.dataset_pu', None)
73	>
74	>	tmp = cfg_params['CMSSW.datasetpath']
75	>	log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
76	>
77	>	if tmp =='':
78	>	msg = "Error: datasetpath not defined "
79	>	raise CrabException(msg)
80	>	elif string.lower(tmp)=='none':
81	>	self.datasetPath = None
82	>	self.selectNoInput = 1
83		else:
84	<	try:
85	<	datasetpath_split = self.datasetPath.split("/")
75	<	# standard style
76	<	if self.use_dbs_1 == 1 :
77	<	self.setParam_('dataset', datasetpath_split[1])
78	<	self.setParam_('owner', datasetpath_split[-1])
79	<	else:
80	<	self.setParam_('dataset', datasetpath_split[1])
81	<	self.setParam_('owner', datasetpath_split[2])
82	<	except:
83	<	self.setParam_('dataset', self.datasetPath)
84	<	self.setParam_('owner', self.datasetPath)
85	<
86	<	self.setTaskid_()
87	<	self.setParam_('taskId', self.cfg_params['taskId'])
84	>	self.datasetPath = tmp
85	>	self.selectNoInput = 0
86
87		self.dataTiers = []
88	<
88	>	self.debugWrap = ''
89	>	self.debug_wrapper = cfg_params.get('USER.debug_wrapper',False)
90	>	if self.debug_wrapper: self.debugWrap='--debug'
91		## now the application
92	<	try:
93	<	self.executable = cfg_params['CMSSW.executable']
94	<	self.setParam_('exe', self.executable)
95	<	log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable)
96	<	msg = "Default executable cmsRun overridden. Switch to " + self.executable
97	<	log.debug(3,msg)
98	<	except KeyError:
99	<	self.executable = 'cmsRun'
100	<	self.setParam_('exe', self.executable)
101	<	msg = "User executable not defined. Use cmsRun"
102	<	log.debug(3,msg)
103	<	pass
92	>	self.executable = cfg_params.get('CMSSW.executable','cmsRun')
93	>	log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable)
94
95	<	try:
106	<	self.pset = cfg_params['CMSSW.pset']
107	<	log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset)
108	<	if self.pset.lower() != 'none' :
109	<	if (not os.path.exists(self.pset)):
110	<	raise CrabException("User defined PSet file "+self.pset+" does not exist")
111	<	else:
112	<	self.pset = None
113	<	except KeyError:
95	>	if not cfg_params.has_key('CMSSW.pset'):
96		raise CrabException("PSet file missing. Cannot run cmsRun ")
97	+	self.pset = cfg_params['CMSSW.pset']
98	+	log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset)
99	+	if self.pset.lower() != 'none' :
100	+	if (not os.path.exists(self.pset)):
101	+	raise CrabException("User defined PSet file "+self.pset+" does not exist")
102	+	else:
103	+	self.pset = None
104
105		# output files
106		## stuff which must be returned always via sandbox
#	Line 121 \| Line 110 \| class Cmssw(JobType):
110		self.output_file_sandbox.append(self.fjrFileName)
111
112		# other output files to be returned via sandbox or copied to SE
113	<	try:
114	<	self.output_file = []
115	<	tmp = cfg_params['CMSSW.output_file']
116	<	if tmp != '':
117	<	tmpOutFiles = string.split(cfg_params['CMSSW.output_file'],',')
118	<	log.debug(7, 'cmssw::cmssw(): output files '+str(tmpOutFiles))
119	<	for tmp in tmpOutFiles:
120	<	tmp=string.strip(tmp)
132	<	self.output_file.append(tmp)
133	<	pass
134	<	else:
135	<	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
136	<	pass
137	<	pass
138	<	except KeyError:
139	<	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
140	<	pass
113	>	outfileflag = False
114	>	self.output_file = []
115	>	tmp = cfg_params.get('CMSSW.output_file',None)
116	>	if tmp :
117	>	self.output_file = [x.strip() for x in tmp.split(',')]
118	>	outfileflag = True #output found
119	>	#else:
120	>	# log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
121
122		# script_exe file as additional file in inputSandbox
123	<	try:
124	<	self.scriptExe = cfg_params['USER.script_exe']
125	<	if self.scriptExe != '':
126	<	if not os.path.isfile(self.scriptExe):
127	<	msg ="ERROR. file "+self.scriptExe+" not found"
128	<	raise CrabException(msg)
149	<	self.additional_inbox_files.append(string.strip(self.scriptExe))
150	<	except KeyError:
151	<	self.scriptExe = ''
123	>	self.scriptExe = cfg_params.get('USER.script_exe',None)
124	>	if self.scriptExe :
125	>	if not os.path.isfile(self.scriptExe):
126	>	msg ="ERROR. file "+self.scriptExe+" not found"
127	>	raise CrabException(msg)
128	>	self.additional_inbox_files.append(string.strip(self.scriptExe))
129
153	–	#CarlosDaniele
130		if self.datasetPath == None and self.pset == None and self.scriptExe == '' :
131	<	msg ="Error. script_exe not defined"
132	<	raise CrabException(msg)
131	>	msg ="Error. script_exe not defined"
132	>	raise CrabException(msg)
133	>
134	>	# use parent files...
135	>	self.useParent = self.cfg_params.get('CMSSW.use_parent',False)
136
137		## additional input files
138	<	try:
138	>	if cfg_params.has_key('USER.additional_input_files'):
139		tmpAddFiles = string.split(cfg_params['USER.additional_input_files'],',')
140		for tmp in tmpAddFiles:
141		tmp = string.strip(tmp)
#	Line 173 \| Line 152 \| class Cmssw(JobType):
152		if not os.path.exists(file):
153		raise CrabException("Additional input file not found: "+file)
154		pass
176	–	# fname = string.split(file, '/')[-1]
177	–	# storedFile = common.work_space.pathForTgz()+'share/'+fname
178	–	# shutil.copyfile(file, storedFile)
155		self.additional_inbox_files.append(string.strip(file))
156		pass
157		pass
158		common.logger.debug(5,"Additional input files: "+str(self.additional_inbox_files))
159	<	except KeyError:
184	<	pass
185	<
186	<	# files per job
187	<	try:
188	<	if (cfg_params['CMSSW.files_per_jobs']):
189	<	raise CrabException("files_per_jobs no longer supported. Quitting.")
190	<	except KeyError:
191	<	pass
159	>	pass
160
161		## Events per job
162	<	try:
162	>	if cfg_params.has_key('CMSSW.events_per_job'):
163		self.eventsPerJob =int( cfg_params['CMSSW.events_per_job'])
164		self.selectEventsPerJob = 1
165	<	except KeyError:
165	>	else:
166		self.eventsPerJob = -1
167		self.selectEventsPerJob = 0
168	<
168	>
169		## number of jobs
170	<	try:
170	>	if cfg_params.has_key('CMSSW.number_of_jobs'):
171		self.theNumberOfJobs =int( cfg_params['CMSSW.number_of_jobs'])
172		self.selectNumberOfJobs = 1
173	<	except KeyError:
173	>	else:
174		self.theNumberOfJobs = 0
175		self.selectNumberOfJobs = 0
176
177	<	try:
177	>	if cfg_params.has_key('CMSSW.total_number_of_events'):
178		self.total_number_of_events = int(cfg_params['CMSSW.total_number_of_events'])
179		self.selectTotalNumberEvents = 1
180	<	except KeyError:
180	>	if self.selectNumberOfJobs == 1:
181	>	if (self.total_number_of_events != -1) and int(self.total_number_of_events) < int(self.theNumberOfJobs):
182	>	msg = 'Must specify at least one event per job. total_number_of_events > number_of_jobs '
183	>	raise CrabException(msg)
184	>	else:
185		self.total_number_of_events = 0
186		self.selectTotalNumberEvents = 0
187
188	<	if self.pset != None: #CarlosDaniele
188	>	if self.pset != None:
189		if ( (self.selectTotalNumberEvents + self.selectEventsPerJob + self.selectNumberOfJobs) != 2 ):
190		msg = 'Must define exactly two of total_number_of_events, events_per_job, or number_of_jobs.'
191		raise CrabException(msg)
#	Line 222 \| Line 194 \| class Cmssw(JobType):
194		msg = 'Must specify number_of_jobs.'
195		raise CrabException(msg)
196
197	<	## source seed for pythia
198	<	try:
199	<	self.sourceSeed = int(cfg_params['CMSSW.pythia_seed'])
200	<	except KeyError:
201	<	self.sourceSeed = None
202	<	common.logger.debug(5,"No seed given")
203	<
204	<	try:
205	<	self.sourceSeedVtx = int(cfg_params['CMSSW.vtx_seed'])
206	<	except KeyError:
207	<	self.sourceSeedVtx = None
208	<	common.logger.debug(5,"No vertex seed given")
197	>	## New method of dealing with seeds
198	>	self.incrementSeeds = []
199	>	self.preserveSeeds = []
200	>	if cfg_params.has_key('CMSSW.preserve_seeds'):
201	>	tmpList = cfg_params['CMSSW.preserve_seeds'].split(',')
202	>	for tmp in tmpList:
203	>	tmp.strip()
204	>	self.preserveSeeds.append(tmp)
205	>	if cfg_params.has_key('CMSSW.increment_seeds'):
206	>	tmpList = cfg_params['CMSSW.increment_seeds'].split(',')
207	>	for tmp in tmpList:
208	>	tmp.strip()
209	>	self.incrementSeeds.append(tmp)
210	>
211	>	## FUTURE: Can remove in CRAB 2.4.0
212	>	self.sourceSeed = cfg_params.get('CMSSW.pythia_seed',None)
213	>	self.sourceSeedVtx = cfg_params.get('CMSSW.vtx_seed',None)
214	>	self.sourceSeedG4 = cfg_params.get('CMSSW.g4_seed',None)
215	>	self.sourceSeedMix = cfg_params.get('CMSSW.mix_seed',None)
216	>	if self.sourceSeed or self.sourceSeedVtx or self.sourceSeedG4 or self.sourceSeedMix:
217	>	msg = 'pythia_seed, vtx_seed, g4_seed, and mix_seed are no longer valid settings. You must use increment_seeds or preserve_seeds'
218	>	raise CrabException(msg)
219
220	<	try:
239	<	self.sourceSeedG4 = int(cfg_params['CMSSW.g4_seed'])
240	<	except KeyError:
241	<	self.sourceSeedG4 = None
242	<	common.logger.debug(5,"No g4 sim hits seed given")
220	>	self.firstRun = cfg_params.get('CMSSW.first_run',None)
221
222	<	try:
223	<	self.sourceSeedMix = int(cfg_params['CMSSW.mix_seed'])
224	<	except KeyError:
247	<	self.sourceSeedMix = None
248	<	common.logger.debug(5,"No mix seed given")
249	<
250	<	try:
251	<	self.firstRun = int(cfg_params['CMSSW.first_run'])
252	<	except KeyError:
253	<	self.firstRun = None
254	<	common.logger.debug(5,"No first run given")
255	<	if self.pset != None: #CarlosDaniele
256	<	ver = string.split(self.version,"_")
257	<	if (int(ver[1])>=1 and int(ver[2])>=5):
258	<	import PsetManipulator150 as pp
259	<	else:
260	<	import PsetManipulator as pp
261	<	PsetEdit = pp.PsetManipulator(self.pset) #Daniele Pset
222	>	# Copy/return
223	>	self.copy_data = int(cfg_params.get('USER.copy_data',0))
224	>	self.return_data = int(cfg_params.get('USER.return_data',0))
225
226		#DBSDLS-start
227	<	## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
227	>	## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
228		self.maxEvents=0 # max events available ( --> check the requested nb. of evts in Creator.py)
229		self.DBSPaths={} # all dbs paths requested ( --> input to the site local discovery script)
230		self.jobDestination=[] # Site destination(s) for each job (list of lists)
#	Line 270 \| Line 233 \| class Cmssw(JobType):
233		blockSites = {}
234		if self.datasetPath:
235		blockSites = self.DataDiscoveryAndLocation(cfg_params)
236	<	#DBSDLS-end
236	>	#DBSDLS-end
237
275	–	self.tgzNameWithPath = self.getTarBall(self.executable)
276	–
238		## Select Splitting
239	<	if self.selectNoInput:
240	<	if self.pset == None: #CarlosDaniele
239	>	if self.selectNoInput:
240	>	if self.pset == None:
241		self.jobSplittingForScript()
242		else:
243		self.jobSplittingNoInput()
244	+	elif (cfg_params.get('CMSSW.noblockboundary',0)):
245	+	self.jobSplittingNoBlockBoundary(blockSites)
246		else:
247		self.jobSplittingByBlocks(blockSites)
248
249	<	# modify Pset
250	<	if self.pset != None: #CarlosDaniele
251	<	try:
252	<	if (self.datasetPath): # standard job
253	<	# allow to processa a fraction of events in a file
254	<	PsetEdit.inputModule("INPUT")
255	<	PsetEdit.maxEvent("INPUTMAXEVENTS")
256	<	PsetEdit.skipEvent("INPUTSKIPEVENTS")
257	<	else: # pythia like job
249	>	# modify Pset only the first time
250	>	if isNew:
251	>	if self.pset != None:
252	>	import PsetManipulator as pp
253	>	PsetEdit = pp.PsetManipulator(self.pset)
254	>	try:
255	>	# Add FrameworkJobReport to parameter-set, set max events.
256	>	# Reset later for data jobs by writeCFG which does all modifications
257	>	PsetEdit.addCrabFJR(self.fjrFileName) # FUTURE: Job report addition not needed by CMSSW>1.5
258		PsetEdit.maxEvent(self.eventsPerJob)
259	<	if (self.firstRun):
260	<	PsetEdit.pythiaFirstRun("INPUTFIRSTRUN") #First Run
261	<	if (self.sourceSeed) :
262	<	PsetEdit.pythiaSeed("INPUT")
263	<	if (self.sourceSeedVtx) :
264	<	PsetEdit.vtxSeed("INPUTVTX")
265	<	if (self.sourceSeedG4) :
266	<	self.PsetEdit.g4Seed("INPUTG4")
267	<	if (self.sourceSeedMix) :
268	<	self.PsetEdit.mixSeed("INPUTMIX")
269	<	# add FrameworkJobReport to parameter-set
270	<	PsetEdit.addCrabFJR(self.fjrFileName)
271	<	PsetEdit.psetWriter(self.configFilename())
272	<	except:
273	<	msg='Error while manipuliating ParameterSet: exiting...'
274	<	raise CrabException(msg)
259	>	PsetEdit.psetWriter(self.configFilename())
260	>	## If present, add TFileService to output files
261	>	if not int(cfg_params.get('CMSSW.skip_TFileService_output',0)):
262	>	tfsOutput = PsetEdit.getTFileService()
263	>	if tfsOutput:
264	>	if tfsOutput in self.output_file:
265	>	common.logger.debug(5,"Output from TFileService "+tfsOutput+" already in output files")
266	>	else:
267	>	outfileflag = True #output found
268	>	self.output_file.append(tfsOutput)
269	>	common.logger.message("Adding "+tfsOutput+" to output files (from TFileService)")
270	>	pass
271	>	pass
272	>	## If present and requested, add PoolOutputModule to output files
273	>	if int(cfg_params.get('CMSSW.get_edm_output',0)):
274	>	edmOutput = PsetEdit.getPoolOutputModule()
275	>	if edmOutput:
276	>	if edmOutput in self.output_file:
277	>	common.logger.debug(5,"Output from PoolOutputModule "+edmOutput+" already in output files")
278	>	else:
279	>	self.output_file.append(edmOutput)
280	>	common.logger.message("Adding "+edmOutput+" to output files (from PoolOutputModule)")
281	>	pass
282	>	pass
283	>	except CrabException:
284	>	msg='Error while manipulating ParameterSet: exiting...'
285	>	raise CrabException(msg)
286	>	## Prepare inputSandbox TarBall (only the first time)
287	>	self.tgzNameWithPath = self.getTarBall(self.executable)
288
289		def DataDiscoveryAndLocation(self, cfg_params):
290
291		import DataDiscovery
316	–	import DataDiscovery_DBS2
292		import DataLocation
293		common.logger.debug(10,"CMSSW::DataDiscoveryAndLocation()")
294
#	Line 322 \| Line 297 \| class Cmssw(JobType):
297		## Contact the DBS
298		common.logger.message("Contacting Data Discovery Services ...")
299		try:
300	<
326	<	if self.use_dbs_1 == 1 :
327	<	self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params)
328	<	else :
329	<	self.pubdata=DataDiscovery_DBS2.DataDiscovery_DBS2(datasetPath, cfg_params)
300	>	self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params,self.skip_blocks)
301		self.pubdata.fetchDBSInfo()
302
303		except DataDiscovery.NotExistingDatasetError, ex :
#	Line 338 \| Line 309 \| class Cmssw(JobType):
309		except DataDiscovery.DataDiscoveryError, ex:
310		msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
311		raise CrabException(msg)
341	–	except DataDiscovery_DBS2.NotExistingDatasetError_DBS2, ex :
342	–	msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
343	–	raise CrabException(msg)
344	–	except DataDiscovery_DBS2.NoDataTierinProvenanceError_DBS2, ex :
345	–	msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
346	–	raise CrabException(msg)
347	–	except DataDiscovery_DBS2.DataDiscoveryError_DBS2, ex:
348	–	msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
349	–	raise CrabException(msg)
312
313		self.filesbyblock=self.pubdata.getFiles()
314		self.eventsbyblock=self.pubdata.getEventsPerBlock()
315		self.eventsbyfile=self.pubdata.getEventsPerFile()
316	+	self.parentFiles=self.pubdata.getParent()
317
318		## get max number of events
319	<	self.maxEvents=self.pubdata.getMaxEvents() ## self.maxEvents used in Creator.py
319	>	self.maxEvents=self.pubdata.getMaxEvents()
320
321		## Contact the DLS and build a list of sites hosting the fileblocks
322		try:
#	Line 362 \| Line 325 \| class Cmssw(JobType):
325		except DataLocation.DataLocationError , ex:
326		msg = 'ERROR ***: failed Data Location in DLS \n %s '%ex.getErrorMessage()
327		raise CrabException(msg)
328	<
328	>
329
330		sites = dataloc.getSites()
331		allSites = []
#	Line 376 \| Line 339 \| class Cmssw(JobType):
339		common.logger.message("Requested dataset: " + datasetPath + " has " + str(self.maxEvents) + " events in " + str(len(self.filesbyblock.keys())) + " blocks.\n")
340
341		return sites
342	<
342	>
343		def jobSplittingByBlocks(self, blockSites):
344		"""
345		Perform job splitting. Jobs run over an integer number of files
#	Line 426 \| Line 389 \| class Cmssw(JobType):
389		totalNumberOfJobs = 999999999
390		else :
391		totalNumberOfJobs = self.ncjobs
429	–
392
393		blocks = blockSites.keys()
394		blockCount = 0
#	Line 446 \| Line 408 \| class Cmssw(JobType):
408		blockCount += 1
409		if block not in jobsOfBlock.keys() :
410		jobsOfBlock[block] = []
411	<
411	>
412		if self.eventsbyblock.has_key(block) :
413		numEventsInBlock = self.eventsbyblock[block]
414		common.logger.debug(5,'Events in Block File '+str(numEventsInBlock))
415	<
415	>
416		files = self.filesbyblock[block]
417		numFilesInBlock = len(files)
418		if (numFilesInBlock <= 0):
#	Line 458 \| Line 420 \| class Cmssw(JobType):
420		fileCount = 0
421
422		# ---- New block => New job ---- #
423	<	parString = "\\{"
423	>	parString = ""
424		# counter for number of events in files currently worked on
425		filesEventCount = 0
426		# flag if next while loop should touch new file
427		newFile = 1
428		# job event counter
429		jobSkipEventCount = 0
430	<
430	>
431		# ---- Iterate over the files in the block until we've met the requested ---- #
432		# ---- total # of events or we've gone over all the files in this block ---- #
433	+	pString=''
434		while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) and (jobCount < totalNumberOfJobs) ):
435		file = files[fileCount]
436	+	if self.useParent:
437	+	parent = self.parentFiles[file]
438	+	for f in parent :
439	+	pString += '\\\"' + f + '\\\"\,'
440	+	common.logger.debug(6, "File "+str(file)+" has the following parents: "+str(parent))
441	+	common.logger.write("File "+str(file)+" has the following parents: "+str(parent))
442		if newFile :
443		try:
444		numEventsInFile = self.eventsbyfile[file]
#	Line 481 \| Line 450 \| class Cmssw(JobType):
450		newFile = 0
451		except KeyError:
452		common.logger.message("File "+str(file)+" has unknown number of events: skipping")
484	–
453
454	+	eventsPerJobRequested = min(eventsPerJobRequested, eventsRemaining)
455		# if less events in file remain than eventsPerJobRequested
456	<	if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested ) :
456	>	if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested):
457		# if last file in block
458		if ( fileCount == numFilesInBlock-1 ) :
459		# end job using last file, use remaining events in block
460		# close job and touch new file
461		fullString = parString[:-2]
462	<	fullString += '\\}'
463	<	list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
462	>	if self.useParent:
463	>	fullParentString = pString[:-2]
464	>	list_of_lists.append([fullString,fullParentString,str(-1),str(jobSkipEventCount)])
465	>	else:
466	>	list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
467		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(filesEventCount - jobSkipEventCount)+" events (last file in block).")
468		self.jobDestination.append(blockSites[block])
469		common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
#	Line 503 \| Line 475 \| class Cmssw(JobType):
475		eventsRemaining = eventsRemaining - filesEventCount + jobSkipEventCount
476		jobSkipEventCount = 0
477		# reset file
478	<	parString = "\\{"
478	>	pString = ""
479	>	parString = ""
480		filesEventCount = 0
481		newFile = 1
482		fileCount += 1
#	Line 515 \| Line 488 \| class Cmssw(JobType):
488		elif ( filesEventCount - jobSkipEventCount == eventsPerJobRequested ) :
489		# close job and touch new file
490		fullString = parString[:-2]
491	<	fullString += '\\}'
492	<	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
491	>	if self.useParent:
492	>	fullParentString = pString[:-2]
493	>	list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount)])
494	>	else:
495	>	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
496		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
497		self.jobDestination.append(blockSites[block])
498		common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
#	Line 527 \| Line 503 \| class Cmssw(JobType):
503		eventsRemaining = eventsRemaining - eventsPerJobRequested
504		jobSkipEventCount = 0
505		# reset file
506	<	parString = "\\{"
506	>	pString = ""
507	>	parString = ""
508		filesEventCount = 0
509		newFile = 1
510		fileCount += 1
511	<
511	>
512		# if more events in file remain than eventsPerJobRequested
513		else :
514		# close job but don't touch new file
515		fullString = parString[:-2]
516	<	fullString += '\\}'
517	<	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
516	>	if self.useParent:
517	>	fullParentString = pString[:-2]
518	>	list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount)])
519	>	else:
520	>	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
521		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
522		self.jobDestination.append(blockSites[block])
523		common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
#	Line 551 \| Line 531 \| class Cmssw(JobType):
531		jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file])
532		# remove all but the last file
533		filesEventCount = self.eventsbyfile[file]
534	<	parString = "\\{"
535	<	parString += '\\\"' + file + '\\\"\,'
534	>	if self.useParent:
535	>	for f in parent : pString += '\\\"' + f + '\\\"\,'
536	>	parString = '\\\"' + file + '\\\"\,'
537		pass # END if
538		pass # END while (iterate over files in the block)
539		pass # END while (iterate over blocks in the dataset)
#	Line 560 \| Line 541 \| class Cmssw(JobType):
541		if (eventsRemaining > 0 and jobCount < totalNumberOfJobs ):
542		common.logger.message("Could not run on all requested events because some blocks not hosted at allowed sites.")
543		common.logger.message(str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n")
544	<
544	>
545		# screen output
546		screenOutput = "List of jobs and available destination sites:\n\n"
547
548	+	# keep trace of block with no sites to print a warning at the end
549	+	noSiteBlock = []
550	+	bloskNoSite = []
551	+
552		blockCounter = 0
553		for block in blocks:
554		if block in jobsOfBlock.keys() :
555		blockCounter += 1
556	<	screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),','.join(blockSites[block]))
556	>	screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),
557	>	','.join(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)))
558	>	if len(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)) == 0:
559	>	noSiteBlock.append( spanRanges(jobsOfBlock[block]) )
560	>	bloskNoSite.append( blockCounter )
561
562		common.logger.message(screenOutput)
563	+	if len(noSiteBlock) > 0 and len(bloskNoSite) > 0:
564	+	msg = 'WARNING: No sites are hosting any part of data for block:\n '
565	+	virgola = ""
566	+	if len(bloskNoSite) > 1:
567	+	virgola = ","
568	+	for block in bloskNoSite:
569	+	msg += ' ' + str(block) + virgola
570	+	msg += '\n Related jobs:\n '
571	+	virgola = ""
572	+	if len(noSiteBlock) > 1:
573	+	virgola = ","
574	+	for range_jobs in noSiteBlock:
575	+	msg += str(range_jobs) + virgola
576	+	msg += '\n will not be submitted and this block of data can not be analyzed!\n'
577	+	if self.cfg_params.has_key('EDG.se_white_list'):
578	+	msg += 'WARNING: SE White List: '+self.cfg_params['EDG.se_white_list']+'\n'
579	+	msg += '(Hint: By whitelisting you force the job to run at this particular site(s).\n'
580	+	msg += 'Please check if the dataset is available at this site!)\n'
581	+	if self.cfg_params.has_key('EDG.ce_white_list'):
582	+	msg += 'WARNING: CE White List: '+self.cfg_params['EDG.ce_white_list']+'\n'
583	+	msg += '(Hint: By whitelisting you force the job to run at this particular site(s).\n'
584	+	msg += 'Please check if the dataset is available at this site!)\n'
585	+
586	+	common.logger.message(msg)
587	+
588	+	self.list_of_args = list_of_lists
589	+	return
590	+
591	+	def jobSplittingNoBlockBoundary(self,blockSites):
592	+	"""
593	+	"""
594	+	# ---- Handle the possible job splitting configurations ---- #
595	+	if (self.selectTotalNumberEvents):
596	+	totalEventsRequested = self.total_number_of_events
597	+	if (self.selectEventsPerJob):
598	+	eventsPerJobRequested = self.eventsPerJob
599	+	if (self.selectNumberOfJobs):
600	+	totalEventsRequested = self.theNumberOfJobs * self.eventsPerJob
601	+
602	+	# If user requested all the events in the dataset
603	+	if (totalEventsRequested == -1):
604	+	eventsRemaining=self.maxEvents
605	+	# If user requested more events than are in the dataset
606	+	elif (totalEventsRequested > self.maxEvents):
607	+	eventsRemaining = self.maxEvents
608	+	common.logger.message("Requested "+str(self.total_number_of_events)+ " events, but only "+str(self.maxEvents)+" events are available.")
609	+	# If user requested less events than are in the dataset
610	+	else:
611	+	eventsRemaining = totalEventsRequested
612	+
613	+	# If user requested more events per job than are in the dataset
614	+	if (self.selectEventsPerJob and eventsPerJobRequested > self.maxEvents):
615	+	eventsPerJobRequested = self.maxEvents
616	+
617	+	# For user info at end
618	+	totalEventCount = 0
619	+
620	+	if (self.selectTotalNumberEvents and self.selectNumberOfJobs):
621	+	eventsPerJobRequested = int(eventsRemaining/self.theNumberOfJobs)
622	+
623	+	if (self.selectNumberOfJobs):
624	+	common.logger.message("May not create the exact number_of_jobs requested.")
625	+
626	+	if ( self.ncjobs == 'all' ) :
627	+	totalNumberOfJobs = 999999999
628	+	else :
629	+	totalNumberOfJobs = self.ncjobs
630	+
631	+	blocks = blockSites.keys()
632	+	blockCount = 0
633	+	# Backup variable in case self.maxEvents counted events in a non-included block
634	+	numBlocksInDataset = len(blocks)
635	+
636	+	jobCount = 0
637	+	list_of_lists = []
638	+
639	+	#AF
640	+	#AF do not reset input files and event count on block boundary
641	+	#AF
642	+	parString=""
643	+	filesEventCount = 0
644	+	#AF
645	+
646	+	# list tracking which jobs are in which jobs belong to which block
647	+	jobsOfBlock = {}
648	+	while ( (eventsRemaining > 0) and (blockCount < numBlocksInDataset) and (jobCount < totalNumberOfJobs)):
649	+	block = blocks[blockCount]
650	+	blockCount += 1
651	+	if block not in jobsOfBlock.keys() :
652	+	jobsOfBlock[block] = []
653	+
654	+	if self.eventsbyblock.has_key(block) :
655	+	numEventsInBlock = self.eventsbyblock[block]
656	+	common.logger.debug(5,'Events in Block File '+str(numEventsInBlock))
657	+	files = self.filesbyblock[block]
658	+	numFilesInBlock = len(files)
659	+	if (numFilesInBlock <= 0):
660	+	continue
661	+	fileCount = 0
662	+	#AF
663	+	#AF do not reset input files and event count of block boundary
664	+	#AF
665	+	## ---- New block => New job ---- #
666	+	#parString = ""
667	+	# counter for number of events in files currently worked on
668	+	#filesEventCount = 0
669	+	#AF
670	+	# flag if next while loop should touch new file
671	+	newFile = 1
672	+	# job event counter
673	+	jobSkipEventCount = 0
674	+
675	+	# ---- Iterate over the files in the block until we've met the requested ---- #
676	+	# ---- total # of events or we've gone over all the files in this block ---- #
677	+	pString=''
678	+	while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) and (jobCount < totalNumberOfJobs) ):
679	+	file = files[fileCount]
680	+	if self.useParent:
681	+	parent = self.parentFiles[file]
682	+	for f in parent :
683	+	pString += '\\\"' + f + '\\\"\,'
684	+	common.logger.debug(6, "File "+str(file)+" has the following parents: "+str(parent))
685	+	common.logger.write("File "+str(file)+" has the following parents: "+str(parent))
686	+	if newFile :
687	+	try:
688	+	numEventsInFile = self.eventsbyfile[file]
689	+	common.logger.debug(6, "File "+str(file)+" has "+str(numEventsInFile)+" events")
690	+	# increase filesEventCount
691	+	filesEventCount += numEventsInFile
692	+	# Add file to current job
693	+	parString += '\\\"' + file + '\\\"\,'
694	+	newFile = 0
695	+	except KeyError:
696	+	common.logger.message("File "+str(file)+" has unknown number of events: skipping")
697	+	eventsPerJobRequested = min(eventsPerJobRequested, eventsRemaining)
698	+	#common.logger.message("AF filesEventCount %s - jobSkipEventCount %s "%(filesEventCount,jobSkipEventCount))
699	+	# if less events in file remain than eventsPerJobRequested
700	+	if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested):
701	+	#AF
702	+	#AF skip fileboundary part
703	+	#AF
704	+	# go to next file
705	+	newFile = 1
706	+	fileCount += 1
707	+	# if events in file equal to eventsPerJobRequested
708	+	elif ( filesEventCount - jobSkipEventCount == eventsPerJobRequested ) :
709	+	# close job and touch new file
710	+	fullString = parString[:-2]
711	+	if self.useParent:
712	+	fullParentString = pString[:-2]
713	+	list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount)])
714	+	else:
715	+	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
716	+	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
717	+	self.jobDestination.append(blockSites[block])
718	+	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
719	+	jobsOfBlock[block].append(jobCount+1)
720	+	# reset counter
721	+	jobCount = jobCount + 1
722	+	totalEventCount = totalEventCount + eventsPerJobRequested
723	+	eventsRemaining = eventsRemaining - eventsPerJobRequested
724	+	jobSkipEventCount = 0
725	+	# reset file
726	+	pString = ""
727	+	parString = ""
728	+	filesEventCount = 0
729	+	newFile = 1
730	+	fileCount += 1
731	+
732	+	# if more events in file remain than eventsPerJobRequested
733	+	else :
734	+	# close job but don't touch new file
735	+	fullString = parString[:-2]
736	+	if self.useParent:
737	+	fullParentString = pString[:-2]
738	+	list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount)])
739	+	else:
740	+	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
741	+	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
742	+	self.jobDestination.append(blockSites[block])
743	+	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
744	+	jobsOfBlock[block].append(jobCount+1)
745	+	# increase counter
746	+	jobCount = jobCount + 1
747	+	totalEventCount = totalEventCount + eventsPerJobRequested
748	+	eventsRemaining = eventsRemaining - eventsPerJobRequested
749	+	# calculate skip events for last file
750	+	# use filesEventCount (contains several files), jobSkipEventCount and eventsPerJobRequest
751	+	jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file])
752	+	# remove all but the last file
753	+	filesEventCount = self.eventsbyfile[file]
754	+	if self.useParent:
755	+	for f in parent : pString += '\\\"' + f + '\\\"\,'
756	+	parString = '\\\"' + file + '\\\"\,'
757	+	pass # END if
758	+	pass # END while (iterate over files in the block)
759	+	pass # END while (iterate over blocks in the dataset)
760	+	self.ncjobs = self.total_number_of_jobs = jobCount
761	+	if (eventsRemaining > 0 and jobCount < totalNumberOfJobs ):
762	+	common.logger.message("eventsRemaining "+str(eventsRemaining))
763	+	common.logger.message("jobCount "+str(jobCount))
764	+	common.logger.message(" totalNumberOfJobs "+str(totalNumberOfJobs))
765	+	common.logger.message("Could not run on all requested events because some blocks not hosted at allowed sites.")
766	+	common.logger.message(str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n")
767	+
768	+	# screen output
769	+	screenOutput = "List of jobs and available destination sites:\n\n"
770
771	+	#AF
772	+	#AF skip check on block with no sites
773	+	#AF
774		self.list_of_args = list_of_lists
775	+
776		return
777
778	+
779	+
780		def jobSplittingNoInput(self):
781		"""
782		Perform job splitting based on number of event per job
783		"""
784		common.logger.debug(5,'Splitting per events')
785	<	common.logger.message('Required '+str(self.eventsPerJob)+' events per job ')
786	<	common.logger.message('Required '+str(self.theNumberOfJobs)+' jobs in total ')
787	<	common.logger.message('Required '+str(self.total_number_of_events)+' events in total ')
785	>
786	>	if (self.selectEventsPerJob):
787	>	common.logger.message('Required '+str(self.eventsPerJob)+' events per job ')
788	>	if (self.selectNumberOfJobs):
789	>	common.logger.message('Required '+str(self.theNumberOfJobs)+' jobs in total ')
790	>	if (self.selectTotalNumberEvents):
791	>	common.logger.message('Required '+str(self.total_number_of_events)+' events in total ')
792
793		if (self.total_number_of_events < 0):
794		msg='Cannot split jobs per Events with "-1" as total number of events'
#	Line 591 \| Line 797 \| class Cmssw(JobType):
797		if (self.selectEventsPerJob):
798		if (self.selectTotalNumberEvents):
799		self.total_number_of_jobs = int(self.total_number_of_events/self.eventsPerJob)
800	<	elif(self.selectNumberOfJobs) :
800	>	elif(self.selectNumberOfJobs) :
801		self.total_number_of_jobs =self.theNumberOfJobs
802	<	self.total_number_of_events =int(self.theNumberOfJobs*self.eventsPerJob)
802	>	self.total_number_of_events =int(self.theNumberOfJobs*self.eventsPerJob)
803
804		elif (self.selectNumberOfJobs) :
805		self.total_number_of_jobs = self.theNumberOfJobs
806		self.eventsPerJob = int(self.total_number_of_events/self.total_number_of_jobs)
807	<
807	>
808		common.logger.debug(5,'N jobs '+str(self.total_number_of_jobs))
809
810		# is there any remainder?
#	Line 614 \| Line 820 \| class Cmssw(JobType):
820		self.list_of_args = []
821		for i in range(self.total_number_of_jobs):
822		## Since there is no input, any site is good
823	<	# self.jobDestination.append(["Any"])
618	<	self.jobDestination.append([""]) #must be empty to write correctly the xml
823	>	self.jobDestination.append([""]) #must be empty to write correctly the xml
824		args=[]
825		if (self.firstRun):
826	<	## pythia first run
622	<	#self.list_of_args.append([(str(self.firstRun)+str(i))])
826	>	## pythia first run
827		args.append(str(self.firstRun)+str(i))
624	–	else:
625	–	## no first run
626	–	#self.list_of_args.append([str(i)])
627	–	args.append(str(i))
628	–	if (self.sourceSeed):
629	–	args.append(str(self.sourceSeed)+str(i))
630	–	if (self.sourceSeedVtx):
631	–	## + vtx random seed
632	–	args.append(str(self.sourceSeedVtx)+str(i))
633	–	if (self.sourceSeedG4):
634	–	## + G4 random seed
635	–	args.append(str(self.sourceSeedG4)+str(i))
636	–	if (self.sourceSeedMix):
637	–	## + Mix random seed
638	–	args.append(str(self.sourceSeedMix)+str(i))
639	–	pass
640	–	pass
828		self.list_of_args.append(args)
642	–	pass
643	–
644	–	# print self.list_of_args
829
830		return
831
832
833	<	def jobSplittingForScript(self):#CarlosDaniele
833	>	def jobSplittingForScript(self):
834		"""
835		Perform job splitting based on number of job
836		"""
#	Line 662 \| Line 846 \| class Cmssw(JobType):
846		# argument is seed number.$i
847		self.list_of_args = []
848		for i in range(self.total_number_of_jobs):
665	–	## Since there is no input, any site is good
666	–	# self.jobDestination.append(["Any"])
849		self.jobDestination.append([""])
668	–	## no random seed
850		self.list_of_args.append([str(i)])
851		return
852
853	<	def split(self, jobParams):
854	<
674	<	common.jobDB.load()
675	<	#### Fabio
853	>	def split(self, jobParams,firstJobID):
854	>
855		njobs = self.total_number_of_jobs
856		arglist = self.list_of_args
857		# create the empty structure
858		for i in range(njobs):
859		jobParams.append("")
681	–
682	–	for job in range(njobs):
683	–	jobParams[job] = arglist[job]
684	–	# print str(arglist[job])
685	–	# print jobParams[job]
686	–	common.jobDB.setArguments(job, jobParams[job])
687	–	common.logger.debug(5,"Job "+str(job)+" Destination: "+str(self.jobDestination[job]))
688	–	common.jobDB.setDestination(job, self.jobDestination[job])
860
861	<	common.jobDB.save()
861	>	listID=[]
862	>	listField=[]
863	>	for id in range(njobs):
864	>	job = id + int(firstJobID)
865	>	jobParams[id] = arglist[id]
866	>	listID.append(job+1)
867	>	job_ToSave ={}
868	>	concString = ' '
869	>	argu=''
870	>	if len(jobParams[id]):
871	>	argu += concString.join(jobParams[id] )
872	>	job_ToSave['arguments']= str(job+1)+' '+argu
873	>	job_ToSave['dlsDestination']= self.jobDestination[id]
874	>	listField.append(job_ToSave)
875	>	msg="Job "+str(job)+" Arguments: "+str(job+1)+" "+argu+"\n" \
876	>	+" Destination: "+str(self.jobDestination[id])
877	>	common.logger.debug(5,msg)
878	>	common._db.updateJob_(listID,listField)
879	>	self.argsList = (len(jobParams[0])+1)
880	>
881		return
882	<
693	<	def getJobTypeArguments(self, nj, sched):
694	<	result = ''
695	<	for i in common.jobDB.arguments(nj):
696	<	result=result+str(i)+" "
697	<	return result
698	<
882	>
883		def numberOfJobs(self):
700	–	# Fabio
884		return self.total_number_of_jobs
885
886		def getTarBall(self, exe):
887		"""
888		Return the TarBall with lib and exe
889		"""
890	<
708	<	# if it exist, just return it
709	<	#
710	<	# Marco. Let's start to use relative path for Boss XML files
711	<	#
712	<	self.tgzNameWithPath = common.work_space.pathForTgz()+'share/'+self.tgz_name
890	>	self.tgzNameWithPath = common.work_space.pathForTgz()+self.tgz_name
891		if os.path.exists(self.tgzNameWithPath):
892		return self.tgzNameWithPath
893
#	Line 722 \| Line 900 \| class Cmssw(JobType):
900
901		# First of all declare the user Scram area
902		swArea = self.scram.getSWArea_()
725	–	#print "swArea = ", swArea
726	–	# swVersion = self.scram.getSWVersion()
727	–	# print "swVersion = ", swVersion
903		swReleaseTop = self.scram.getReleaseTop_()
904	<	#print "swReleaseTop = ", swReleaseTop
730	<
904	>
905		## check if working area is release top
906		if swReleaseTop == '' or swArea == swReleaseTop:
907	+	common.logger.debug(3,"swArea = "+swArea+" swReleaseTop ="+swReleaseTop)
908		return
909
910		import tarfile
#	Line 740 \| Line 915 \| class Cmssw(JobType):
915		exeWithPath = self.scram.findFile_(executable)
916		if ( not exeWithPath ):
917		raise CrabException('User executable '+executable+' not found')
918	<
918	>
919		## then check if it's private or not
920		if exeWithPath.find(swReleaseTop) == -1:
921		# the exe is private, so we must ship
#	Line 749 \| Line 924 \| class Cmssw(JobType):
924		# distinguish case when script is in user project area or given by full path somewhere else
925		if exeWithPath.find(path) >= 0 :
926		exe = string.replace(exeWithPath, path,'')
927	<	tar.add(path+exe,os.path.basename(executable))
927	>	tar.add(path+exe,exe)
928		else :
929		tar.add(exeWithPath,os.path.basename(executable))
930		pass
931		else:
932		# the exe is from release, we'll find it on WN
933		pass
934	<
934	>
935		## Now get the libraries: only those in local working area
936		libDir = 'lib'
937		lib = swArea+'/' +libDir
938		common.logger.debug(5,"lib "+lib+" to be tarred")
939		if os.path.exists(lib):
940		tar.add(lib,libDir)
941	<
941	>
942		## Now check if module dir is present
943		moduleDir = 'module'
944		module = swArea + '/' + moduleDir
#	Line 771 \| Line 946 \| class Cmssw(JobType):
946		tar.add(module,moduleDir)
947
948		## Now check if any data dir(s) is present
949	<	swAreaLen=len(swArea)
950	<	for root, dirs, files in os.walk(swArea):
951	<	if "data" in dirs:
952	<	common.logger.debug(5,"data "+root+"/data"+" to be tarred")
953	<	tar.add(root+"/data",root[swAreaLen:]+"/data")
954	<
955	<	## Add ProdAgent dir to tar
956	<	paDir = 'ProdAgentApi'
957	<	pa = os.environ['CRABDIR'] + '/' + 'ProdAgentApi'
958	<	if os.path.isdir(pa):
959	<	tar.add(pa,paDir)
960	<
961	<	### FEDE FOR DBS PUBLICATION
962	<	## Add PRODCOMMON dir to tar
963	<	prodcommonDir = 'ProdCommon'
964	<	prodcommonPath = os.environ['CRABDIR'] + '/' + 'ProdCommon'
965	<	if os.path.isdir(prodcommonPath):
966	<	tar.add(prodcommonPath,prodcommonDir)
967	<	#############################
968	<
949	>	self.dataExist = False
950	>	todo_list = [(i, i) for i in os.listdir(swArea+"/src")]
951	>	while len(todo_list):
952	>	entry, name = todo_list.pop()
953	>	if name.startswith('crab_0_') or name.startswith('.') or name == 'CVS':
954	>	continue
955	>	if os.path.isdir(swArea+"/src/"+entry):
956	>	entryPath = entry + '/'
957	>	todo_list += [(entryPath + i, i) for i in os.listdir(swArea+"/src/"+entry)]
958	>	if name == 'data':
959	>	self.dataExist=True
960	>	common.logger.debug(5,"data "+entry+" to be tarred")
961	>	tar.add(swArea+"/src/"+entry,"src/"+entry)
962	>	pass
963	>	pass
964	>
965	>	### CMSSW ParameterSet
966	>	if not self.pset is None:
967	>	cfg_file = common.work_space.jobDir()+self.configFilename()
968	>	tar.add(cfg_file,self.configFilename())
969	>	common.logger.debug(5,"File added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
970	>
971	>
972	>	## Add ProdCommon dir to tar
973	>	prodcommonDir = './'
974	>	prodcommonPath = os.environ['CRABDIR'] + '/' + 'external/'
975	>	neededStuff = ['ProdCommon/__init__.py','ProdCommon/FwkJobRep', 'ProdCommon/CMSConfigTools', \
976	>	'ProdCommon/Core', 'ProdCommon/MCPayloads', 'IMProv', 'ProdCommon/Storage']
977	>	for file in neededStuff:
978	>	tar.add(prodcommonPath+file,prodcommonDir+file)
979	>	common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
980	>
981	>	##### ML stuff
982	>	ML_file_list=['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py']
983	>	path=os.environ['CRABDIR'] + '/python/'
984	>	for file in ML_file_list:
985	>	tar.add(path+file,file)
986	>	common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
987	>
988	>	##### Utils
989	>	Utils_file_list=['parseCrabFjr.py','writeCfg.py', 'fillCrabFjr.py','cmscp.py']
990	>	for file in Utils_file_list:
991	>	tar.add(path+file,file)
992		common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
993	+
994	+	##### AdditionalFiles
995	+	tar.dereference=True
996	+	for file in self.additional_inbox_files:
997	+	tar.add(file,string.split(file,'/')[-1])
998	+	tar.dereference=False
999	+	common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
1000	+
1001		tar.close()
1002	<	except :
1003	<	raise CrabException('Could not create tar-ball')
1002	>	except IOError, exc:
1003	>	common.logger.write(str(exc))
1004	>	raise CrabException('Could not create tar-ball '+self.tgzNameWithPath)
1005	>	except tarfile.TarError, exc:
1006	>	common.logger.write(str(exc))
1007	>	raise CrabException('Could not create tar-ball '+self.tgzNameWithPath)
1008
1009		## check for tarball size
1010		tarballinfo = os.stat(self.tgzNameWithPath)
1011		if ( tarballinfo.st_size > self.MaxTarBallSize10241024 ) :
1012	<	raise CrabException('Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' + str(self.MaxTarBallSize) + ' MB input sandbox limit and not supported by the used GRID submission system. Please make sure that no unnecessary files are in all data directories in your local CMSSW project area as they are automatically packed into the input sandbox.')
1012	>	msg = 'Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' + str(self.MaxTarBallSize) \
1013	>	+'MB input sandbox limit \n'
1014	>	msg += ' and not supported by the direct GRID submission system.\n'
1015	>	msg += ' Please use the CRAB server mode by setting server_name=<NAME> in section [CRAB] of your crab.cfg.\n'
1016	>	msg += ' For further infos please see https://twiki.cern.ch/twiki/bin/view/CMS/CrabServer#CRABSERVER_for_Users'
1017	>	raise CrabException(msg)
1018
1019		## create tar-ball with ML stuff
805	–	self.MLtgzfile = common.work_space.pathForTgz()+'share/MLfiles.tgz'
806	–	try:
807	–	tar = tarfile.open(self.MLtgzfile, "w:gz")
808	–	path=os.environ['CRABDIR'] + '/python/'
809	–	for file in ['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py', 'parseCrabFjr.py']:
810	–	tar.add(path+file,file)
811	–	common.logger.debug(5,"Files added to "+self.MLtgzfile+" : "+str(tar.getnames()))
812	–	tar.close()
813	–	except :
814	–	raise CrabException('Could not create ML files tar-ball')
815	–
816	–	return
817	–
818	–	def additionalInputFileTgz(self):
819	–	"""
820	–	Put all additional files into a tar ball and return its name
821	–	"""
822	–	import tarfile
823	–	tarName= common.work_space.pathForTgz()+'share/'+self.additional_tgz_name
824	–	tar = tarfile.open(tarName, "w:gz")
825	–	for file in self.additional_inbox_files:
826	–	tar.add(file,string.split(file,'/')[-1])
827	–	common.logger.debug(5,"Files added to "+self.additional_tgz_name+" : "+str(tar.getnames()))
828	–	tar.close()
829	–	return tarName
1020
1021	<	def wsSetupEnvironment(self, nj):
1021	>	def wsSetupEnvironment(self, nj=0):
1022		"""
1023		Returns part of a job script which prepares
1024		the execution environment for the job 'nj'.
1025		"""
1026	+	if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
1027	+	psetName = 'pset.py'
1028	+	else:
1029	+	psetName = 'pset.cfg'
1030		# Prepare JobType-independent part
1031	<	txt = ''
1032	<
1033	<	## OLI_Daniele at this level middleware already known
840	<
841	<	txt += 'echo "### Firtst set SCRAM ARCH and BUILD_ARCH ###"\n'
842	<	txt += 'echo "Setting SCRAM_ARCH='+self.executable_arch+'"\n'
843	<	txt += 'export SCRAM_ARCH='+self.executable_arch+'\n'
844	<	txt += 'export BUILD_ARCH='+self.executable_arch+'\n'
845	<	txt += 'if [ $middleware == LCG ]; then \n'
1031	>	txt = '\n#Written by cms_cmssw::wsSetupEnvironment\n'
1032	>	txt += 'echo ">>> setup environment"\n'
1033	>	txt += 'if [ $middleware == LCG ]; then \n'
1034		txt += self.wsSetupCMSLCGEnvironment_()
1035		txt += 'elif [ $middleware == OSG ]; then\n'
1036		txt += ' WORKING_DIR=`/bin/mktemp -d $OSG_WN_TMP/cms_XXXXXXXXXXXX`\n'
1037	<	txt += ' echo "Created working directory: $WORKING_DIR"\n'
1038	<	txt += ' if [ ! -d $WORKING_DIR ] ;then\n'
1039	<	txt += ' echo "SET_CMS_ENV 10016 ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
1040	<	txt += ' echo "JOB_EXIT_STATUS = 10016"\n'
853	<	txt += ' echo "JobExitCode=10016" \| tee -a $RUNTIME_AREA/$repo\n'
854	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
855	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
856	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
857	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
858	<	txt += ' exit 1\n'
1037	>	txt += ' if [ ! $? == 0 ] ;then\n'
1038	>	txt += ' echo "ERROR ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
1039	>	txt += ' job_exit_code=10016\n'
1040	>	txt += ' func_exit\n'
1041		txt += ' fi\n'
1042	+	txt += ' echo ">>> Created working directory: $WORKING_DIR"\n'
1043		txt += '\n'
1044		txt += ' echo "Change to working directory: $WORKING_DIR"\n'
1045		txt += ' cd $WORKING_DIR\n'
1046	<	txt += self.wsSetupCMSOSGEnvironment_()
1046	>	txt += ' echo ">>> current directory (WORKING_DIR): $WORKING_DIR"\n'
1047	>	txt += self.wsSetupCMSOSGEnvironment_()
1048		txt += 'fi\n'
1049
1050		# Prepare JobType-specific part
1051		scram = self.scram.commandName()
1052		txt += '\n\n'
1053	<	txt += 'echo "### SPECIFIC JOB SETUP ENVIRONMENT ###"\n'
1053	>	txt += 'echo ">>> specific cmssw setup environment:"\n'
1054	>	txt += 'echo "CMSSW_VERSION = '+self.version+'"\n'
1055		txt += scram+' project CMSSW '+self.version+'\n'
1056		txt += 'status=$?\n'
1057		txt += 'if [ $status != 0 ] ; then\n'
1058	<	txt += ' echo "SET_EXE_ENV 10034 ==>ERROR CMSSW '+self.version+' not found on `hostname`" \n'
1059	<	txt += ' echo "JOB_EXIT_STATUS = 10034"\n'
1060	<	txt += ' echo "JobExitCode=10034" \| tee -a $RUNTIME_AREA/$repo\n'
876	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
877	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
878	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
879	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
880	<	## OLI_Daniele
881	<	txt += ' if [ $middleware == OSG ]; then \n'
882	<	txt += ' echo "Remove working directory: $WORKING_DIR"\n'
883	<	txt += ' cd $RUNTIME_AREA\n'
884	<	txt += ' /bin/rm -rf $WORKING_DIR\n'
885	<	txt += ' if [ -d $WORKING_DIR ] ;then\n'
886	<	txt += ' echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
887	<	txt += ' echo "JOB_EXIT_STATUS = 10018"\n'
888	<	txt += ' echo "JobExitCode=10018" \| tee -a $RUNTIME_AREA/$repo\n'
889	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
890	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
891	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
892	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
893	<	txt += ' fi\n'
894	<	txt += ' fi \n'
895	<	txt += ' exit 1 \n'
1058	>	txt += ' echo "ERROR ==> CMSSW '+self.version+' not found on `hostname`" \n'
1059	>	txt += ' job_exit_code=10034\n'
1060	>	txt += ' func_exit\n'
1061		txt += 'fi \n'
897	–	txt += 'echo "CMSSW_VERSION = '+self.version+'"\n'
1062		txt += 'cd '+self.version+'\n'
899	–	########## FEDE FOR DBS2 ######################
1063		txt += 'SOFTWARE_DIR=`pwd`\n'
1064	<	txt += 'echo SOFTWARE_DIR=$SOFTWARE_DIR \n'
902	<	###############################################
903	<	### needed grep for bug in scramv1 ###
904	<	txt += scram+' runtime -sh\n'
1064	>	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
1065		txt += 'eval `'+scram+' runtime -sh \| grep -v SCRAMRT_LSB_JOBNAME`\n'
1066	<	txt += 'echo $PATH\n'
1067	<
1066	>	txt += 'if [ $? != 0 ] ; then\n'
1067	>	txt += ' echo "ERROR ==> Problem with the command: "\n'
1068	>	txt += ' echo "eval \`'+scram+' runtime -sh \| grep -v SCRAMRT_LSB_JOBNAME \` at `hostname`"\n'
1069	>	txt += ' job_exit_code=10034\n'
1070	>	txt += ' func_exit\n'
1071	>	txt += 'fi \n'
1072		# Handle the arguments:
1073		txt += "\n"
1074		txt += "## number of arguments (first argument always jobnumber)\n"
1075		txt += "\n"
1076	<	# txt += "narg=$#\n"
913	<	txt += "if [ $nargs -lt 2 ]\n"
1076	>	txt += "if [ $nargs -lt "+str(self.argsList)+" ]\n"
1077		txt += "then\n"
1078	<	txt += " echo 'SET_EXE_ENV 1 ==> ERROR Too few arguments' +$nargs+ \n"
1079	<	txt += ' echo "JOB_EXIT_STATUS = 50113"\n'
1080	<	txt += ' echo "JobExitCode=50113" \| tee -a $RUNTIME_AREA/$repo\n'
918	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
919	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
920	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
921	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
922	<	## OLI_Daniele
923	<	txt += ' if [ $middleware == OSG ]; then \n'
924	<	txt += ' echo "Remove working directory: $WORKING_DIR"\n'
925	<	txt += ' cd $RUNTIME_AREA\n'
926	<	txt += ' /bin/rm -rf $WORKING_DIR\n'
927	<	txt += ' if [ -d $WORKING_DIR ] ;then\n'
928	<	txt += ' echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
929	<	txt += ' echo "JOB_EXIT_STATUS = 50114"\n'
930	<	txt += ' echo "JobExitCode=50114" \| tee -a $RUNTIME_AREA/$repo\n'
931	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
932	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
933	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
934	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
935	<	txt += ' fi\n'
936	<	txt += ' fi \n'
937	<	txt += " exit 1\n"
1078	>	txt += " echo 'ERROR ==> Too few arguments' +$nargs+ \n"
1079	>	txt += ' job_exit_code=50113\n'
1080	>	txt += " func_exit\n"
1081		txt += "fi\n"
1082		txt += "\n"
1083
1084		# Prepare job-specific part
1085		job = common.job_list[nj]
1086	<	### FEDE FOR DBS OUTPUT PUBLICATION
1087	<	if (self.datasetPath):
1086	>	if (self.datasetPath):
1087	>	self.primaryDataset = self.datasetPath.split("/")[1]
1088	>	DataTier = self.datasetPath.split("/")[2]
1089		txt += '\n'
1090		txt += 'DatasetPath='+self.datasetPath+'\n'
1091
1092	<	datasetpath_split = self.datasetPath.split("/")
1093	<
950	<	txt += 'PrimaryDataset='+datasetpath_split[1]+'\n'
951	<	txt += 'DataTier='+datasetpath_split[2]+'\n'
952	<	#txt += 'ProcessedDataset='+datasetpath_split[3]+'\n'
1092	>	txt += 'PrimaryDataset='+self.primaryDataset +'\n'
1093	>	txt += 'DataTier='+DataTier+'\n'
1094		txt += 'ApplicationFamily=cmsRun\n'
1095
1096		else:
1097	+	self.primaryDataset = 'null'
1098		txt += 'DatasetPath=MCDataTier\n'
1099		txt += 'PrimaryDataset=null\n'
1100		txt += 'DataTier=null\n'
959	–	#txt += 'ProcessedDataset=null\n'
1101		txt += 'ApplicationFamily=MCDataTier\n'
1102	<	if self.pset != None: #CarlosDaniele
1102	>	if self.pset != None:
1103		pset = os.path.basename(job.configFilename())
1104		txt += '\n'
1105		txt += 'cp $RUNTIME_AREA/'+pset+' .\n'
1106		if (self.datasetPath): # standard job
1107	<	#txt += 'InputFiles=$2\n'
1108	<	txt += 'InputFiles=${args[1]}\n'
1109	<	txt += 'MaxEvents=${args[2]}\n'
1110	<	txt += 'SkipEvents=${args[3]}\n'
1107	>	txt += 'InputFiles=${args[1]}; export InputFiles\n'
1108	>	if (self.useParent):
1109	>	txt += 'ParentFiles=${args[2]}; export ParentFiles\n'
1110	>	txt += 'MaxEvents=${args[3]}; export MaxEvents\n'
1111	>	txt += 'SkipEvents=${args[4]}; export SkipEvents\n'
1112	>	else:
1113	>	txt += 'MaxEvents=${args[2]}; export MaxEvents\n'
1114	>	txt += 'SkipEvents=${args[3]}; export SkipEvents\n'
1115		txt += 'echo "Inputfiles:<$InputFiles>"\n'
1116	<	txt += 'sed "s#{\'INPUT\'}#$InputFiles#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1116	>	if (self.useParent): txt += 'echo "ParentFiles:<$ParentFiles>"\n'
1117		txt += 'echo "MaxEvents:<$MaxEvents>"\n'
973	–	txt += 'sed "s#INPUTMAXEVENTS#$MaxEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1118		txt += 'echo "SkipEvents:<$SkipEvents>"\n'
975	–	txt += 'sed "s#INPUTSKIPEVENTS#$SkipEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1119		else: # pythia like job
1120	<	seedIndex=1
1120	>	txt += 'PreserveSeeds=' + ','.join(self.preserveSeeds) + '; export PreserveSeeds\n'
1121	>	txt += 'IncrementSeeds=' + ','.join(self.incrementSeeds) + '; export IncrementSeeds\n'
1122	>	txt += 'echo "PreserveSeeds: <$PreserveSeeds>"\n'
1123	>	txt += 'echo "IncrementSeeds:<$IncrementSeeds>"\n'
1124		if (self.firstRun):
1125	<	txt += 'FirstRun=${args['+str(seedIndex)+']}\n'
1125	>	txt += 'FirstRun=${args[1]}; export FirstRun\n'
1126		txt += 'echo "FirstRun: <$FirstRun>"\n'
981	–	txt += 'sed "s#\<INPUTFIRSTRUN\>#$FirstRun#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
982	–	seedIndex=seedIndex+1
1127
1128	<	if (self.sourceSeed):
985	<	txt += 'Seed=${args['+str(seedIndex)+']}\n'
986	<	txt += 'sed "s#\<INPUT\>#$Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
987	<	seedIndex=seedIndex+1
988	<	## the following seeds are not always present
989	<	if (self.sourceSeedVtx):
990	<	txt += 'VtxSeed=${args['+str(seedIndex)+']}\n'
991	<	txt += 'echo "VtxSeed: <$VtxSeed>"\n'
992	<	txt += 'sed "s#\<INPUTVTX\>#$VtxSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
993	<	seedIndex += 1
994	<	if (self.sourceSeedG4):
995	<	txt += 'G4Seed=${args['+str(seedIndex)+']}\n'
996	<	txt += 'echo "G4Seed: <$G4Seed>"\n'
997	<	txt += 'sed "s#\<INPUTG4\>#$G4Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
998	<	seedIndex += 1
999	<	if (self.sourceSeedMix):
1000	<	txt += 'mixSeed=${args['+str(seedIndex)+']}\n'
1001	<	txt += 'echo "MixSeed: <$mixSeed>"\n'
1002	<	txt += 'sed "s#\<INPUTMIX\>#$mixSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1003	<	seedIndex += 1
1004	<	pass
1005	<	pass
1006	<	txt += 'mv -f '+pset+' pset.cfg\n'
1128	>	txt += 'mv -f ' + pset + ' ' + psetName + '\n'
1129
1008	–	if len(self.additional_inbox_files) > 0:
1009	–	txt += 'if [ -e $RUNTIME_AREA/'+self.additional_tgz_name+' ] ; then\n'
1010	–	txt += ' tar xzvf $RUNTIME_AREA/'+self.additional_tgz_name+'\n'
1011	–	txt += 'fi\n'
1012	–	pass
1130
1131	<	if self.pset != None: #CarlosDaniele
1132	<	txt += 'echo "### END JOB SETUP ENVIRONMENT ###"\n\n'
1016	<
1017	<	txt += '\n'
1018	<	txt += 'echo "*** cat pset.cfg *******"\n'
1019	<	txt += 'cat pset.cfg\n'
1020	<	txt += 'echo "**** end pset.cfg ******"\n'
1131	>	if self.pset != None:
1132	>	# FUTURE: Can simply for 2_1_x and higher
1133		txt += '\n'
1134	<	### FEDE FOR DBS OUTPUT PUBLICATION
1135	<	txt += 'PSETHASH=`EdmConfigHash < pset.cfg` \n'
1134	>	if self.debug_wrapper==True:
1135	>	txt += 'echo "*** cat ' + psetName + ' *******"\n'
1136	>	txt += 'cat ' + psetName + '\n'
1137	>	txt += 'echo "**** end ' + psetName + ' ******"\n'
1138	>	txt += '\n'
1139	>	if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
1140	>	txt += 'PSETHASH=`edmConfigHash ' + psetName + '` \n'
1141	>	else:
1142	>	txt += 'PSETHASH=`edmConfigHash < ' + psetName + '` \n'
1143		txt += 'echo "PSETHASH = $PSETHASH" \n'
1025	–	##############
1144		txt += '\n'
1027	–	# txt += 'echo "*** cat pset1.cfg *******"\n'
1028	–	# txt += 'cat pset1.cfg\n'
1029	–	# txt += 'echo "**** end pset1.cfg ******"\n'
1145		return txt
1146
1147	<	def wsBuildExe(self, nj=0):
1147	>	def wsUntarSoftware(self, nj=0):
1148		"""
1149		Put in the script the commands to build an executable
1150		or a library.
1151		"""
1152
1153	<	txt = ""
1153	>	txt = '\n#Written by cms_cmssw::wsUntarSoftware\n'
1154
1155		if os.path.isfile(self.tgzNameWithPath):
1156	<	txt += 'echo "tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'"\n'
1156	>	txt += 'echo ">>> tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+' :" \n'
1157		txt += 'tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
1158	+	if self.debug_wrapper:
1159	+	txt += 'ls -Al \n'
1160		txt += 'untar_status=$? \n'
1161		txt += 'if [ $untar_status -ne 0 ]; then \n'
1162	<	txt += ' echo "SET_EXE 1 ==> ERROR Untarring .tgz file failed"\n'
1163	<	txt += ' echo "JOB_EXIT_STATUS = $untar_status" \n'
1164	<	txt += ' echo "JobExitCode=$untar_status" \| tee -a $RUNTIME_AREA/$repo\n'
1048	<	txt += ' if [ $middleware == OSG ]; then \n'
1049	<	txt += ' echo "Remove working directory: $WORKING_DIR"\n'
1050	<	txt += ' cd $RUNTIME_AREA\n'
1051	<	txt += ' /bin/rm -rf $WORKING_DIR\n'
1052	<	txt += ' if [ -d $WORKING_DIR ] ;then\n'
1053	<	txt += ' echo "SET_EXE 50999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Untarring .tgz file failed"\n'
1054	<	txt += ' echo "JOB_EXIT_STATUS = 50999"\n'
1055	<	txt += ' echo "JobExitCode=50999" \| tee -a $RUNTIME_AREA/$repo\n'
1056	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1057	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
1058	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
1059	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
1060	<	txt += ' fi\n'
1061	<	txt += ' fi \n'
1062	<	txt += ' \n'
1063	<	txt += ' exit 1 \n'
1162	>	txt += ' echo "ERROR ==> Untarring .tgz file failed"\n'
1163	>	txt += ' job_exit_code=$untar_status\n'
1164	>	txt += ' func_exit\n'
1165		txt += 'else \n'
1166		txt += ' echo "Successful untar" \n'
1167		txt += 'fi \n'
1168		txt += '\n'
1169	<	txt += 'echo "Include ProdAgentApi and PRODCOMMON in PYTHONPATH"\n'
1169	>	txt += 'echo ">>> Include $RUNTIME_AREA in PYTHONPATH:"\n'
1170		txt += 'if [ -z "$PYTHONPATH" ]; then\n'
1171	<	#### FEDE FOR DBS OUTPUT PUBLICATION
1071	<	txt += ' export PYTHONPATH=$SOFTWARE_DIR/ProdAgentApi:$SOFTWARE_DIR/ProdCommon\n'
1072	<	#txt += ' export PYTHONPATH=`pwd`/ProdAgentApi:`pwd`/ProdCommon\n'
1073	<	#txt += ' export PYTHONPATH=ProdAgentApi\n'
1171	>	txt += ' export PYTHONPATH=$RUNTIME_AREA/\n'
1172		txt += 'else\n'
1173	<	txt += ' export PYTHONPATH=$SOFTWARE_DIR/ProdAgentApi:$SOFTWARE_DIR/ProdCommon:${PYTHONPATH}\n'
1076	<	#txt += ' export PYTHONPATH=`pwd`/ProdAgentApi:`pwd`/ProdCommon:${PYTHONPATH}\n'
1077	<	#txt += ' export PYTHONPATH=ProdAgentApi:${PYTHONPATH}\n'
1173	>	txt += ' export PYTHONPATH=$RUNTIME_AREA/:${PYTHONPATH}\n'
1174		txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
1079	–	###################
1175		txt += 'fi\n'
1176		txt += '\n'
1177
1178		pass
1179	<
1179	>
1180		return txt
1181
1182	<	def modifySteeringCards(self, nj):
1182	>	def wsBuildExe(self, nj=0):
1183		"""
1184	<	modify the card provided by the user,
1185	<	writing a new card into share dir
1184	>	Put in the script the commands to build an executable
1185	>	or a library.
1186		"""
1187	<
1187	>
1188	>	txt = '\n#Written by cms_cmssw::wsBuildExe\n'
1189	>	txt += 'echo ">>> moving CMSSW software directories in `pwd`" \n'
1190	>
1191	>	txt += 'rm -r lib/ module/ \n'
1192	>	txt += 'mv $RUNTIME_AREA/lib/ . \n'
1193	>	txt += 'mv $RUNTIME_AREA/module/ . \n'
1194	>	if self.dataExist == True:
1195	>	txt += 'rm -r src/ \n'
1196	>	txt += 'mv $RUNTIME_AREA/src/ . \n'
1197	>	if len(self.additional_inbox_files)>0:
1198	>	for file in self.additional_inbox_files:
1199	>	txt += 'mv $RUNTIME_AREA/'+os.path.basename(file)+' . \n'
1200	>	# txt += 'mv $RUNTIME_AREA/ProdCommon/ . \n'
1201	>	# txt += 'mv $RUNTIME_AREA/IMProv/ . \n'
1202	>
1203	>	txt += 'echo ">>> Include $RUNTIME_AREA in PYTHONPATH:"\n'
1204	>	txt += 'if [ -z "$PYTHONPATH" ]; then\n'
1205	>	txt += ' export PYTHONPATH=$RUNTIME_AREA/\n'
1206	>	txt += 'else\n'
1207	>	txt += ' export PYTHONPATH=$RUNTIME_AREA/:${PYTHONPATH}\n'
1208	>	txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
1209	>	txt += 'fi\n'
1210	>	txt += '\n'
1211	>
1212	>	return txt
1213	>
1214	>
1215		def executableName(self):
1216	<	if self.scriptExe: #CarlosDaniele
1216	>	if self.scriptExe:
1217		return "sh "
1218		else:
1219		return self.executable
1220
1221		def executableArgs(self):
1222	+	# FUTURE: This function tests the CMSSW version. Can be simplified as we drop support for old versions
1223		if self.scriptExe:#CarlosDaniele
1224		return self.scriptExe + " $NJob"
1225		else:
1226	<	# if >= CMSSW_1_5_X, add -e
1227	<	version_array = self.scram.getSWVersion().split('_')
1228	<	major = 0
1229	<	minor = 0
1230	<	try:
1231	<	major = int(version_array[1])
1232	<	minor = int(version_array[2])
1233	<	except:
1111	<	msg = "Cannot parse CMSSW version string: " + "_".join(version_array) + " for major and minor release number!"
1112	<	raise CrabException(msg)
1113	<	if major >= 1 and minor >= 5 :
1114	<	return " -e -p pset.cfg"
1226	>	ex_args = ""
1227	>	# FUTURE: This tests the CMSSW version. Can remove code as versions deprecated
1228	>	# Framework job report
1229	>	if (self.CMSSW_major >= 1 and self.CMSSW_minor >= 5) or (self.CMSSW_major >= 2):
1230	>	ex_args += " -j $RUNTIME_AREA/crab_fjr_$NJob.xml"
1231	>	# Type of config file
1232	>	if self.CMSSW_major >= 2 :
1233	>	ex_args += " -p pset.py"
1234		else:
1235	<	return " -p pset.cfg"
1235	>	ex_args += " -p pset.cfg"
1236	>	return ex_args
1237
1238		def inputSandbox(self, nj):
1239		"""
1240		Returns a list of filenames to be put in JDL input sandbox.
1241		"""
1242		inp_box = []
1123	–	# # dict added to delete duplicate from input sandbox file list
1124	–	# seen = {}
1125	–	## code
1243		if os.path.isfile(self.tgzNameWithPath):
1244		inp_box.append(self.tgzNameWithPath)
1245	<	if os.path.isfile(self.MLtgzfile):
1129	<	inp_box.append(self.MLtgzfile)
1130	<	## config
1131	<	if not self.pset is None:
1132	<	inp_box.append(common.work_space.pathForTgz() + 'job/' + self.configFilename())
1133	<	## additional input files
1134	<	tgz = self.additionalInputFileTgz()
1135	<	inp_box.append(tgz)
1245	>	inp_box.append(common.work_space.jobDir() + self.scriptName)
1246		return inp_box
1247
1248		def outputSandbox(self, nj):
#	Line 1143 \| Line 1253 \| class Cmssw(JobType):
1253
1254		## User Declared output files
1255		for out in (self.output_file+self.output_file_sandbox):
1256	<	n_out = nj + 1
1257	<	out_box.append(self.numberFile_(out,str(n_out)))
1256	>	n_out = nj + 1
1257	>	out_box.append(numberFile(out,str(n_out)))
1258		return out_box
1259
1150	–	def prepareSteeringCards(self):
1151	–	"""
1152	–	Make initial modifications of the user's steering card file.
1153	–	"""
1154	–	return
1260
1261		def wsRenameOutput(self, nj):
1262		"""
1263		Returns part of a job script which renames the produced files.
1264		"""
1265
1266	<	txt = '\n'
1267	<	txt += '# directory content\n'
1268	<	txt += 'ls \n'
1266	>	txt = '\n#Written by cms_cmssw::wsRenameOutput\n'
1267	>	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
1268	>	txt += 'echo ">>> current directory content:"\n'
1269	>	if self.debug_wrapper:
1270	>	txt += 'ls -Al\n'
1271	>	txt += '\n'
1272
1273	<	for fileWithSuffix in (self.output_file+self.output_file_sandbox):
1274	<	output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1273	>	for fileWithSuffix in (self.output_file):
1274	>	output_file_num = numberFile(fileWithSuffix, '$NJob')
1275		txt += '\n'
1276		txt += '# check output file\n'
1169	–	# txt += 'ls '+fileWithSuffix+'\n'
1170	–	# txt += 'ls_result=$?\n'
1277		txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1278	<	txt += ' mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1278	>	if (self.copy_data == 1): # For OSG nodes, file is in $WORKING_DIR, should not be moved to $RUNTIME_AREA
1279	>	txt += ' mv '+fileWithSuffix+' '+output_file_num+'\n'
1280	>	txt += ' ln -s `pwd`/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1281	>	else:
1282	>	txt += ' mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1283	>	txt += ' ln -s $RUNTIME_AREA/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1284		txt += 'else\n'
1285	<	txt += ' exit_status=60302\n'
1286	<	txt += ' echo "ERROR: Problem with output file '+fileWithSuffix+'"\n'
1287	<	if common.scheduler.boss_scheduler_name == 'condor_g':
1285	>	txt += ' job_exit_code=60302\n'
1286	>	txt += ' echo "WARNING: Output file '+fileWithSuffix+' not found"\n'
1287	>	if common.scheduler.name().upper() == 'CONDOR_G':
1288		txt += ' if [ $middleware == OSG ]; then \n'
1289		txt += ' echo "prepare dummy output file"\n'
1290		txt += ' echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
#	Line 1181 \| Line 1292 \| class Cmssw(JobType):
1292		txt += 'fi\n'
1293		file_list = []
1294		for fileWithSuffix in (self.output_file):
1295	<	file_list.append(self.numberFile_(fileWithSuffix, '$NJob'))
1185	<	txt += 'file_list="'+string.join(file_list,' ')+'"\n'
1186	<
1187	<	txt += 'cd $RUNTIME_AREA\n'
1188	<	#### FEDE this is the cleanEnv function
1189	<	### OLI_DANIELE
1190	<	#txt += 'if [ $middleware == OSG ]; then\n'
1191	<	#txt += ' cd $RUNTIME_AREA\n'
1192	<	#txt += ' echo "Remove working directory: $WORKING_DIR"\n'
1193	<	#txt += ' /bin/rm -rf $WORKING_DIR\n'
1194	<	#txt += ' if [ -d $WORKING_DIR ] ;then\n'
1195	<	#txt += ' echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1196	<	#txt += ' echo "JOB_EXIT_STATUS = 60999"\n'
1197	<	#txt += ' echo "JobExitCode=60999" \| tee -a $RUNTIME_AREA/$repo\n'
1198	<	#txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1199	<	#txt += ' rm -f $RUNTIME_AREA/$repo \n'
1200	<	#txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
1201	<	#txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
1202	<	#txt += ' fi\n'
1203	<	#txt += 'fi\n'
1204	<	#txt += '\n'
1205	<
1295	>	file_list.append(numberFile('$SOFTWARE_DIR/'+fileWithSuffix, '$NJob'))
1296
1297	+	txt += 'file_list="'+string.join(file_list,',')+'"\n'
1298	+	txt += '\n'
1299	+	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
1300	+	txt += 'echo ">>> current directory content:"\n'
1301	+	if self.debug_wrapper:
1302	+	txt += 'ls -Al\n'
1303	+	txt += '\n'
1304	+	txt += 'cd $RUNTIME_AREA\n'
1305	+	txt += 'echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
1306		return txt
1307
1209	–	def numberFile_(self, file, txt):
1210	–	"""
1211	–	append _'txt' before last extension of a file
1212	–	"""
1213	–	p = string.split(file,".")
1214	–	# take away last extension
1215	–	name = p[0]
1216	–	for x in p[1:-1]:
1217	–	name=name+"."+x
1218	–	# add "_txt"
1219	–	if len(p)>1:
1220	–	ext = p[len(p)-1]
1221	–	result = name + '_' + txt + "." + ext
1222	–	else:
1223	–	result = name + '_' + txt
1224	–
1225	–	return result
1226	–
1308		def getRequirements(self, nj=[]):
1309		"""
1310	<	return job requirements to add to jdl files
1310	>	return job requirements to add to jdl files
1311		"""
1312		req = ''
1313		if self.version:
1314		req='Member("VO-cms-' + \
1315		self.version + \
1316		'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1317	<	## SL add requirement for OS version only if SL4
1237	<	reSL4 = re.compile( r'slc4' )
1238	<	if self.executable_arch and reSL4.search(self.executable_arch):
1317	>	if self.executable_arch:
1318		req+=' && Member("VO-cms-' + \
1319		self.executable_arch + \
1320		'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1321
1322		req = req + ' && (other.GlueHostNetworkAdapterOutboundIP)'
1323	+	if ( common.scheduler.name() == "glitecoll" ) or ( common.scheduler.name() == "glite"):
1324	+	req += ' && other.GlueCEStateStatus == "Production" '
1325
1326		return req
1327
1328		def configFilename(self):
1329		""" return the config filename """
1330	<	return self.name()+'.cfg'
1330	>	# FUTURE: Can remove cfg mode for CMSSW >= 2_1_x
1331	>	if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
1332	>	return self.name()+'.py'
1333	>	else:
1334	>	return self.name()+'.cfg'
1335
1251	–	### OLI_DANIELE
1336		def wsSetupCMSOSGEnvironment_(self):
1337		"""
1338		Returns part of a job script which is prepares
1339		the execution environment and which is common for all CMS jobs.
1340		"""
1341	<	txt = '\n'
1342	<	txt += ' echo "### SETUP CMS OSG ENVIRONMENT ###"\n'
1343	<	txt += ' if [ -f $GRID3_APP_DIR/cmssoft/cmsset_default.sh ] ;then\n'
1344	<	txt += ' # Use $GRID3_APP_DIR/cmssoft/cmsset_default.sh to setup cms software\n'
1345	<	txt += ' export SCRAM_ARCH='+self.executable_arch+'\n'
1346	<	txt += ' source $GRID3_APP_DIR/cmssoft/cmsset_default.sh '+self.version+'\n'
1263	<	txt += ' elif [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n'
1341	>	txt = '\n#Written by cms_cmssw::wsSetupCMSOSGEnvironment_\n'
1342	>	txt += ' echo ">>> setup CMS OSG environment:"\n'
1343	>	txt += ' echo "set SCRAM ARCH to ' + self.executable_arch + '"\n'
1344	>	txt += ' export SCRAM_ARCH='+self.executable_arch+'\n'
1345	>	txt += ' echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
1346	>	txt += ' if [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n'
1347		txt += ' # Use $OSG_APP/cmssoft/cms/cmsset_default.sh to setup cms software\n'
1348	<	txt += ' export SCRAM_ARCH='+self.executable_arch+'\n'
1349	<	txt += ' source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1350	<	txt += ' else\n'
1351	<	txt += ' echo "SET_CMS_ENV 10020 ==> ERROR $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1352	<	txt += ' echo "JOB_EXIT_STATUS = 10020"\n'
1353	<	txt += ' echo "JobExitCode=10020" \| tee -a $RUNTIME_AREA/$repo\n'
1271	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1272	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
1273	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
1274	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
1275	<	txt += ' exit 1\n'
1276	<	txt += '\n'
1277	<	txt += ' echo "Remove working directory: $WORKING_DIR"\n'
1278	<	txt += ' cd $RUNTIME_AREA\n'
1279	<	txt += ' /bin/rm -rf $WORKING_DIR\n'
1280	<	txt += ' if [ -d $WORKING_DIR ] ;then\n'
1281	<	txt += ' echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1282	<	txt += ' echo "JOB_EXIT_STATUS = 10017"\n'
1283	<	txt += ' echo "JobExitCode=10017" \| tee -a $RUNTIME_AREA/$repo\n'
1284	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1285	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
1286	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
1287	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
1288	<	txt += ' fi\n'
1289	<	txt += '\n'
1290	<	txt += ' exit 1\n'
1291	<	txt += ' fi\n'
1348	>	txt += ' source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1349	>	txt += ' else\n'
1350	>	txt += ' echo "ERROR ==> $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1351	>	txt += ' job_exit_code=10020\n'
1352	>	txt += ' func_exit\n'
1353	>	txt += ' fi\n'
1354		txt += '\n'
1355	<	txt += ' echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1356	<	txt += ' echo " END SETUP CMS OSG ENVIRONMENT "\n'
1355	>	txt += ' echo "==> setup cms environment ok"\n'
1356	>	txt += ' echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
1357
1358		return txt
1359	<
1298	<	### OLI_DANIELE
1359	>
1360		def wsSetupCMSLCGEnvironment_(self):
1361		"""
1362		Returns part of a job script which is prepares
1363		the execution environment and which is common for all CMS jobs.
1364		"""
1365	<	txt = ' \n'
1366	<	txt += ' echo " ### SETUP CMS LCG ENVIRONMENT ### "\n'
1367	<	txt += ' if [ ! $VO_CMS_SW_DIR ] ;then\n'
1368	<	txt += ' echo "SET_CMS_ENV 10031 ==> ERROR CMS software dir not found on WN `hostname`"\n'
1369	<	txt += ' echo "JOB_EXIT_STATUS = 10031" \n'
1370	<	txt += ' echo "JobExitCode=10031" \| tee -a $RUNTIME_AREA/$repo\n'
1371	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1372	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
1373	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
1374	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
1375	<	txt += ' exit 1\n'
1376	<	txt += ' else\n'
1377	<	txt += ' echo "Sourcing environment... "\n'
1378	<	txt += ' if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
1379	<	txt += ' echo "SET_CMS_ENV 10020 ==> ERROR cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1380	<	txt += ' echo "JOB_EXIT_STATUS = 10020"\n'
1381	<	txt += ' echo "JobExitCode=10020" \| tee -a $RUNTIME_AREA/$repo\n'
1382	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1383	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
1384	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
1385	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
1386	<	txt += ' exit 1\n'
1387	<	txt += ' fi\n'
1388	<	txt += ' echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1389	<	txt += ' source $VO_CMS_SW_DIR/cmsset_default.sh\n'
1390	<	txt += ' result=$?\n'
1391	<	txt += ' if [ $result -ne 0 ]; then\n'
1331	<	txt += ' echo "SET_CMS_ENV 10032 ==> ERROR problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1332	<	txt += ' echo "JOB_EXIT_STATUS = 10032"\n'
1333	<	txt += ' echo "JobExitCode=10032" \| tee -a $RUNTIME_AREA/$repo\n'
1334	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1335	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
1336	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
1337	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
1338	<	txt += ' exit 1\n'
1339	<	txt += ' fi\n'
1340	<	txt += ' fi\n'
1341	<	txt += ' \n'
1342	<	txt += ' echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1343	<	txt += ' echo "### END SETUP CMS LCG ENVIRONMENT ###"\n'
1365	>	txt = '\n#Written by cms_cmssw::wsSetupCMSLCGEnvironment_\n'
1366	>	txt += ' echo ">>> setup CMS LCG environment:"\n'
1367	>	txt += ' echo "set SCRAM ARCH and BUILD_ARCH to ' + self.executable_arch + ' ###"\n'
1368	>	txt += ' export SCRAM_ARCH='+self.executable_arch+'\n'
1369	>	txt += ' export BUILD_ARCH='+self.executable_arch+'\n'
1370	>	txt += ' if [ ! $VO_CMS_SW_DIR ] ;then\n'
1371	>	txt += ' echo "ERROR ==> CMS software dir not found on WN `hostname`"\n'
1372	>	txt += ' job_exit_code=10031\n'
1373	>	txt += ' func_exit\n'
1374	>	txt += ' else\n'
1375	>	txt += ' echo "Sourcing environment... "\n'
1376	>	txt += ' if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
1377	>	txt += ' echo "ERROR ==> cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1378	>	txt += ' job_exit_code=10020\n'
1379	>	txt += ' func_exit\n'
1380	>	txt += ' fi\n'
1381	>	txt += ' echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1382	>	txt += ' source $VO_CMS_SW_DIR/cmsset_default.sh\n'
1383	>	txt += ' result=$?\n'
1384	>	txt += ' if [ $result -ne 0 ]; then\n'
1385	>	txt += ' echo "ERROR ==> problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1386	>	txt += ' job_exit_code=10032\n'
1387	>	txt += ' func_exit\n'
1388	>	txt += ' fi\n'
1389	>	txt += ' fi\n'
1390	>	txt += ' \n'
1391	>	txt += ' echo "==> setup cms environment ok"\n'
1392		return txt
1393
1394	<	### FEDE FOR DBS OUTPUT PUBLICATION
1347	<	def modifyReport(self, nj):
1394	>	def wsModifyReport(self, nj):
1395		"""
1396	<	insert the part of the script that modifies the FrameworkJob Report
1396	>	insert the part of the script that modifies the FrameworkJob Report
1397		"""
1398	+	txt = '\n#Written by cms_cmssw::wsModifyReport\n'
1399	+	publish_data = int(self.cfg_params.get('USER.publish_data',0))
1400	+	if (publish_data == 1):
1401
1352	–	txt = ''
1353	–	txt += 'echo "Modify Job Report" \n'
1354	–	#txt += 'chmod a+x $RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1355	–	################ FEDE FOR DBS2 #############################################
1356	–	txt += 'chmod a+x $SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1357	–	#############################################################################
1358	–	try:
1359	–	publish_data = int(self.cfg_params['USER.publish_data'])
1360	–	except KeyError:
1361	–	publish_data = 0
1362	–
1363	–	txt += 'if [ -z "$SE" ]; then\n'
1364	–	txt += ' SE="" \n'
1365	–	txt += 'fi \n'
1366	–	txt += 'if [ -z "$SE_PATH" ]; then\n'
1367	–	txt += ' SE_PATH="" \n'
1368	–	txt += 'fi \n'
1369	–	txt += 'echo "SE = $SE"\n'
1370	–	txt += 'echo "SE_PATH = $SE_PATH"\n'
1371	–
1372	–	if (publish_data == 1):
1373	–	#processedDataset = self.cfg_params['USER.processed_datasetname']
1402		processedDataset = self.cfg_params['USER.publish_data_name']
1403	<	txt += 'ProcessedDataset='+processedDataset+'\n'
1404	<	#### LFN=/store/user/<user>/processedDataset_PSETHASH
1405	<	txt += 'if [ "$SE_PATH" == "" ]; then\n'
1406	<	#### FEDE: added slash in LFN ##############
1403	>
1404	>	txt += 'if [ $StageOutExitStatus -eq 0 ]; then\n'
1405	>	txt += ' FOR_LFN=$LFNBaseName\n'
1406	>	txt += 'else\n'
1407		txt += ' FOR_LFN=/copy_problems/ \n'
1408	<	txt += 'else \n'
1409	<	txt += ' tmp=`echo $SE_PATH \| awk -F \'store\' \'{print$2}\'` \n'
1410	<	##### FEDE TO BE CHANGED, BECAUSE STORE IS HARDCODED!!!! ########
1411	<	txt += ' FOR_LFN=/store$tmp \n'
1412	<	txt += 'fi \n'
1408	>	txt += ' SE=""\n'
1409	>	txt += ' SE_PATH=""\n'
1410	>	txt += 'fi\n'
1411	>
1412	>	txt += 'echo ">>> Modify Job Report:" \n'
1413	>	txt += 'chmod a+x $RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py\n'
1414	>	txt += 'ProcessedDataset='+processedDataset+'\n'
1415	>	#txt += 'ProcessedDataset=$procDataset \n'
1416		txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1417	+	txt += 'echo "SE = $SE"\n'
1418	+	txt += 'echo "SE_PATH = $SE_PATH"\n'
1419		txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1420		txt += 'echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n'
1421	<	#txt += 'echo "$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1422	<	txt += 'echo "$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1423	<	txt += '$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1424	<	#txt += '$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1425	<
1421	>	args = '$RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier ' \
1422	>	'$USER-$ProcessedDataset-$PSETHASH $ApplicationFamily '+ \
1423	>	' $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH'
1424	>	txt += 'echo "$RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py '+str(args)+'"\n'
1425	>	txt += '$RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py '+str(args)+'\n'
1426		txt += 'modifyReport_result=$?\n'
1394	–	txt += 'echo modifyReport_result = $modifyReport_result\n'
1427		txt += 'if [ $modifyReport_result -ne 0 ]; then\n'
1428	<	txt += ' exit_status=1\n'
1429	<	txt += ' echo "ERROR: Problem with ModifyJobReport"\n'
1428	>	txt += ' modifyReport_result=70500\n'
1429	>	txt += ' job_exit_code=$modifyReport_result\n'
1430	>	txt += ' echo "ModifyReportResult=$modifyReport_result" \| tee -a $RUNTIME_AREA/$repo\n'
1431	>	txt += ' echo "WARNING: Problem with ModifyJobReport"\n'
1432		txt += 'else\n'
1433	<	txt += ' mv NewFrameworkJobReport.xml crab_fjr_$NJob.xml\n'
1433	>	txt += ' mv NewFrameworkJobReport.xml $RUNTIME_AREA/crab_fjr_$NJob.xml\n'
1434		txt += 'fi\n'
1401	–	else:
1402	–	txt += 'ProcessedDataset=no_data_to_publish \n'
1403	–	#### FEDE: added slash in LFN ##############
1404	–	txt += 'FOR_LFN=/local/ \n'
1405	–	txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1406	–	txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1435		return txt
1436
1437	<	def cleanEnv(self):
1438	<	### OLI_DANIELE
1439	<	txt = ''
1440	<	txt += 'if [ $middleware == OSG ]; then\n'
1441	<	txt += ' cd $RUNTIME_AREA\n'
1442	<	txt += ' echo "Remove working directory: $WORKING_DIR"\n'
1443	<	txt += ' /bin/rm -rf $WORKING_DIR\n'
1444	<	txt += ' if [ -d $WORKING_DIR ] ;then\n'
1445	<	txt += ' echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1446	<	txt += ' echo "JOB_EXIT_STATUS = 60999"\n'
1447	<	txt += ' echo "JobExitCode=60999" \| tee -a $RUNTIME_AREA/$repo\n'
1448	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1449	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
1450	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
1451	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
1437	>	def wsParseFJR(self):
1438	>	"""
1439	>	Parse the FrameworkJobReport to obtain useful infos
1440	>	"""
1441	>	txt = '\n#Written by cms_cmssw::wsParseFJR\n'
1442	>	txt += 'echo ">>> Parse FrameworkJobReport crab_fjr.xml"\n'
1443	>	txt += 'if [ -s $RUNTIME_AREA/crab_fjr_$NJob.xml ]; then\n'
1444	>	txt += ' if [ -s $RUNTIME_AREA/parseCrabFjr.py ]; then\n'
1445	>	txt += ' cmd_out=`python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --dashboard $MonitorID,$MonitorJobID '+self.debugWrap+'`\n'
1446	>	if self.debug_wrapper :
1447	>	txt += ' echo "Result of parsing the FrameworkJobReport crab_fjr.xml: $cmd_out"\n'
1448	>	txt += ' executable_exit_status=`python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --exitcode`\n'
1449	>	txt += ' if [ $executable_exit_status -eq 50115 ];then\n'
1450	>	txt += ' echo ">>> crab_fjr.xml contents: "\n'
1451	>	txt += ' cat $RUNTIME_AREA/crab_fjr_$NJob.xml\n'
1452	>	txt += ' echo "Wrong FrameworkJobReport --> does not contain useful info. ExitStatus: $executable_exit_status"\n'
1453	>	txt += ' elif [ $executable_exit_status -eq -999 ];then\n'
1454	>	txt += ' echo "ExitStatus from FrameworkJobReport not available. not available. Using exit code of executable from command line."\n'
1455	>	txt += ' else\n'
1456	>	txt += ' echo "Extracted ExitStatus from FrameworkJobReport parsing output: $executable_exit_status"\n'
1457	>	txt += ' fi\n'
1458	>	txt += ' else\n'
1459	>	txt += ' echo "CRAB python script to parse CRAB FrameworkJobReport crab_fjr.xml is not available, using exit code of executable from command line."\n'
1460	>	txt += ' fi\n'
1461	>	#### Patch to check input data reading for CMSSW16x Hopefully we-ll remove it asap
1462	>	txt += ' if [ $executable_exit_status -eq 0 ];then\n'
1463	>	txt += ' echo ">>> Executable succeded $executable_exit_status"\n'
1464	>	if (self.datasetPath and not (self.dataset_pu or self.useParent)) :
1465	>	# VERIFY PROCESSED DATA
1466	>	txt += ' echo ">>> Verify list of processed files:"\n'
1467	>	txt += ' echo $InputFiles \|tr -d \'\\\\\' \|tr \',\' \'\\n\'\|tr -d \'"\' > input-files.txt\n'
1468	>	txt += ' python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --lfn > processed-files.txt\n'
1469	>	txt += ' cat input-files.txt \| sort \| uniq > tmp.txt\n'
1470	>	txt += ' mv tmp.txt input-files.txt\n'
1471	>	txt += ' echo "cat input-files.txt"\n'
1472	>	txt += ' echo "----------------------"\n'
1473	>	txt += ' cat input-files.txt\n'
1474	>	txt += ' cat processed-files.txt \| sort \| uniq > tmp.txt\n'
1475	>	txt += ' mv tmp.txt processed-files.txt\n'
1476	>	txt += ' echo "----------------------"\n'
1477	>	txt += ' echo "cat processed-files.txt"\n'
1478	>	txt += ' echo "----------------------"\n'
1479	>	txt += ' cat processed-files.txt\n'
1480	>	txt += ' echo "----------------------"\n'
1481	>	txt += ' diff -q input-files.txt processed-files.txt\n'
1482	>	txt += ' fileverify_status=$?\n'
1483	>	txt += ' if [ $fileverify_status -ne 0 ]; then\n'
1484	>	txt += ' executable_exit_status=30001\n'
1485	>	txt += ' echo "ERROR ==> not all input files processed"\n'
1486	>	txt += ' echo " ==> list of processed files from crab_fjr.xml differs from list in pset.cfg"\n'
1487	>	txt += ' echo " ==> diff input-files.txt processed-files.txt"\n'
1488	>	txt += ' fi\n'
1489	>	txt += ' elif [ $executable_exit_status -ne 0 ] \|\| [ $executable_exit_status -ne 50015 ] \|\| [ $executable_exit_status -ne 50017 ];then\n'
1490	>	txt += ' echo ">>> Executable failed $executable_exit_status"\n'
1491	>	txt += ' echo "ExeExitCode=$executable_exit_status" \| tee -a $RUNTIME_AREA/$repo\n'
1492	>	txt += ' echo "EXECUTABLE_EXIT_STATUS = $executable_exit_status"\n'
1493	>	txt += ' job_exit_code=$executable_exit_status\n'
1494	>	txt += ' func_exit\n'
1495		txt += ' fi\n'
1496	+	txt += '\n'
1497	+	txt += 'else\n'
1498	+	txt += ' echo "CRAB FrameworkJobReport crab_fjr.xml is not available, using exit code of executable from command line."\n'
1499		txt += 'fi\n'
1500		txt += '\n'
1501	+	txt += 'echo "ExeExitCode=$executable_exit_status" \| tee -a $RUNTIME_AREA/$repo\n'
1502	+	txt += 'echo "EXECUTABLE_EXIT_STATUS = $executable_exit_status"\n'
1503	+	txt += 'job_exit_code=$executable_exit_status\n'
1504	+
1505		return txt
1506
1507		def setParam_(self, param, value):
#	Line 1432 \| Line 1510 \| class Cmssw(JobType):
1510		def getParams(self):
1511		return self._params
1512
1435	–	def setTaskid_(self):
1436	–	self._taskId = self.cfg_params['taskId']
1437	–
1438	–	def getTaskid(self):
1439	–	return self._taskId
1440	–
1513		def uniquelist(self, old):
1514		"""
1515		remove duplicates from a list
#	Line 1446 \| Line 1518 \| class Cmssw(JobType):
1518		for e in old:
1519		nd[e]=0
1520		return nd.keys()
1521	+
1522	+	def outList(self):
1523	+	"""
1524	+	check the dimension of the output files
1525	+	"""
1526	+	txt = ''
1527	+	txt += 'echo ">>> list of expected files on output sandbox"\n'
1528	+	listOutFiles = []
1529	+	stdout = 'CMSSW_$NJob.stdout'
1530	+	stderr = 'CMSSW_$NJob.stderr'
1531	+	if (self.return_data == 1):
1532	+	for file in (self.output_file+self.output_file_sandbox):
1533	+	listOutFiles.append(numberFile(file, '$NJob'))
1534	+	listOutFiles.append(stdout)
1535	+	listOutFiles.append(stderr)
1536	+	else:
1537	+	for file in (self.output_file_sandbox):
1538	+	listOutFiles.append(numberFile(file, '$NJob'))
1539	+	listOutFiles.append(stdout)
1540	+	listOutFiles.append(stderr)
1541	+	txt += 'echo "output files: '+string.join(listOutFiles,' ')+'"\n'
1542	+	txt += 'filesToCheck="'+string.join(listOutFiles,' ')+'"\n'
1543	+	txt += 'export filesToCheck\n'
1544	+	return txt

Diff Legend

-–
+Removed lines
-+
+Added lines
-<
+Changed lines
->
+Changed lines

Comparing COMP/CRAB/python/cms_cmssw.py (file contents): Revision 1.107 by gutsche, Tue Jul 24 19:49:35 2007 UTC vs. Revision 1.254 by ewv, Thu Oct 30 16:25:24 2008 UTC

Diff Legend

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.107 by gutsche, Tue Jul 24 19:49:35 2007 UTC vs.
Revision 1.254 by ewv, Thu Oct 30 16:25:24 2008 UTC