ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/cms_cmssw.py

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.14 by gutsche, Tue Jun 27 15:39:58 2006 UTC vs.
Revision 1.365 by spiga, Tue Nov 9 21:10:07 2010 UTC

#	Line 1 | Line 1
1	+
2	+	__revision__ = "$Id$"
3	+	__version__ = "$Revision$"
4	+
5		from JobType import JobType
2	–	from crab_logger import Logger
6		from crab_exceptions import *
7		from crab_util import *
8		import common
9	<	import PsetManipulator
7	<
8	<	import DBSInfo_EDM
9	<	import DataDiscovery_EDM
10	<	import DataLocation_EDM
9	>	import re
10		import Scram
11	<
12	<	import os, string, re
11	>	from Splitter import JobSplitter
12	>	from Downloader import Downloader
13	>	try:
14	>	import json
15	>	except:
16	>	import simplejson as json
17	>
18	>	from IMProv.IMProvNode import IMProvNode
19	>	from IMProv.IMProvLoader import loadIMProvFile
20	>	import os, string, glob
21	>	from xml.dom import pulldom
22
23		class Cmssw(JobType):
24	<	def __init__(self, cfg_params):
24	>	def __init__(self, cfg_params, ncjobs,skip_blocks, isNew):
25		JobType.__init__(self, 'CMSSW')
26	<	common.logger.debug(3,'CMSSW::__init__')
27	<
28	<	self.analisys_common_info = {}
29	<	# Marco.
26	>	common.logger.debug('CMSSW::__init__')
27	>	self.skip_blocks = skip_blocks
28	>	self.argsList = 2
29	>	self.NumEvents=0
30		self._params = {}
31		self.cfg_params = cfg_params
32	<	log = common.logger
33	<
32	>	### FEDE FOR MULTI ###
33	>	self.var_filter=''
34	>
35	>	### Temporary patch to automatically skip the ISB size check:
36	>	self.server = self.cfg_params.get('CRAB.server_name',None) or \
37	>	self.cfg_params.get('CRAB.use_server',0)
38	>	self.local  = common.scheduler.name().upper() in ['LSF','CAF','CONDOR','SGE','PBS']
39	>	size = 9.5
40	>	if self.server :
41	>	size = 1000
42	>	elif self.local:
43	>	size = 9999999
44	>	self.MaxTarBallSize = float(self.cfg_params.get('GRID.maxtarballsize',size))
45	>
46	>	# number of jobs requested to be created, limit obj splitting
47	>	self.ncjobs = ncjobs
48	>
49		self.scram = Scram.Scram(cfg_params)
27	–	scramArea = ''
50		self.additional_inbox_files = []
51		self.scriptExe = ''
52		self.executable = ''
53	+	self.executable_arch = self.scram.getArch()
54		self.tgz_name = 'default.tgz'
55	<
55	>	self.scriptName = 'CMSSW.sh'
56	>	self.pset = ''
57	>	self.datasetPath = ''
58	>
59	>	self.tgzNameWithPath = common.work_space.pathForTgz()+self.tgz_name
60	>	# set FJR file name
61	>	self.fjrFileName = 'crab_fjr.xml'
62
63		self.version = self.scram.getSWVersion()
64	<	self.setParam_('application', self.version)
65	<	common.analisys_common_info['sw_version'] = self.version
66	<	### FEDE
67	<	common.analisys_common_info['copy_input_data'] = 0
68	<	common.analisys_common_info['events_management'] = 1
40	<
41	<	### collect Data cards
64	>	common.logger.log(10-1,"CMSSW version is: "+str(self.version))
65	>	version_array = self.version.split('_')
66	>	self.CMSSW_major = 0
67	>	self.CMSSW_minor = 0
68	>	self.CMSSW_patch = 0
69		try:
70	<	tmp =  cfg_params['CMSSW.datasetpath']
71	<	log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
72	<	if string.lower(tmp)=='none':
73	<	self.datasetPath = None
74	<	else:
48	<	self.datasetPath = tmp
49	<	except KeyError:
50	<	msg = "Error: datasetpath not defined "
70	>	self.CMSSW_major = int(version_array[1])
71	>	self.CMSSW_minor = int(version_array[2])
72	>	self.CMSSW_patch = int(version_array[3])
73	>	except:
74	>	msg = "Cannot parse CMSSW version string: " + self.version + " for major and minor release number!"
75		raise CrabException(msg)
76
77	<	# ML monitoring
78	<	# split dataset path style: /PreProdR3Minbias/SIM/GEN-SIM
79	<	if not self.datasetPath:
80	<	self.setParam_('dataset', 'None')
81	<	self.setParam_('owner', 'None')
82	<	else:
83	<	datasetpath_split = self.datasetPath.split("/")
84	<	self.setParam_('dataset', datasetpath_split[1])
85	<	self.setParam_('owner', datasetpath_split[-1])
77	>	if self.CMSSW_major < 2 or (self.CMSSW_major == 2 and self.CMSSW_minor < 1):
78	>	msg = "CRAB supports CMSSW >= 2_1_x only. Use an older CRAB version."
79	>	raise CrabException(msg)
80	>	"""
81	>	As CMSSW versions are dropped we can drop more code:
82	>	2.x dropped: drop check for lumi range setting
83	>	"""
84	>	self.checkCMSSWVersion()
85	>	### collect Data cards
86
87	<	self.setTaskid_()
88	<	self.setParam_('taskId', self.cfg_params['taskId'])
87	>	### Temporary: added to remove input file control in the case of PU
88	>	self.dataset_pu = cfg_params.get('CMSSW.dataset_pu', None)
89
90	<	self.dataTiers = []
90	>	tmp =  cfg_params['CMSSW.datasetpath']
91	>	common.logger.log(10-1, "CMSSW::CMSSW(): datasetPath = "+tmp)
92	>
93	>	if tmp =='':
94	>	msg = "Error: datasetpath not defined "
95	>	raise CrabException(msg)
96	>	elif string.lower(tmp)=='none':
97	>	self.datasetPath = None
98	>	self.selectNoInput = 1
99	>	self.primaryDataset = 'null'
100	>	else:
101	>	self.datasetPath = tmp
102	>	self.selectNoInput = 0
103	>	ll = len(self.datasetPath.split("/"))
104	>	if (ll < 4):
105	>	msg = 'Your datasetpath has a invalid format ' + self.datasetPath + '\n'
106	>	msg += 'Expected a path in format /PRIMARY/PROCESSED/TIER1-TIER2 or /PRIMARY/PROCESSED/TIER/METHOD for ADS'
107	>	raise CrabException(msg)
108	>	self.primaryDataset = self.datasetPath.split("/")[1]
109	>	self.dataTier = self.datasetPath.split("/")[2]
110	>
111	>	# Analysis dataset is primary/processed/tier/definition
112	>	self.ads = False
113	>	if self.datasetPath:
114	>	self.ads = len(self.datasetPath.split("/")) > 4
115	>	self.lumiMask = self.cfg_params.get('CMSSW.lumi_mask',None)
116	>	self.lumiParams = self.cfg_params.get('CMSSW.total_number_of_lumis',None) or \
117	>	self.cfg_params.get('CMSSW.lumis_per_job',None)
118	>
119	>	# FUTURE: Can remove this check
120	>	if self.ads and self.CMSSW_major < 3:
121	>	common.logger.info('Warning: Analysis dataset support is incomplete in CMSSW 2_x.')
122	>	common.logger.info('  Only file level, not lumi level, granularity is supported.')
123	>
124	>	self.debugWrap=''
125	>	self.debug_wrapper = int(cfg_params.get('USER.debug_wrapper',0))
126	>	if self.debug_wrapper == 1: self.debugWrap='--debug'
127
128		## now the application
129	<	try:
130	<	self.executable = cfg_params['CMSSW.executable']
131	<	self.setParam_('exe', self.executable)
132	<	log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable)
73	<	msg = "Default executable cmsRun overridden. Switch to " + self.executable
74	<	log.debug(3,msg)
75	<	except KeyError:
76	<	self.executable = 'cmsRun'
77	<	self.setParam_('exe', self.executable)
78	<	msg = "User executable not defined. Use cmsRun"
79	<	log.debug(3,msg)
80	<	pass
129	>	self.managedGenerators = ['madgraph', 'comphep', 'lhe']
130	>	self.generator = cfg_params.get('CMSSW.generator','pythia').lower()
131	>	self.executable = cfg_params.get('CMSSW.executable','cmsRun')
132	>	common.logger.log(10-1, "CMSSW::CMSSW(): executable = "+self.executable)
133
134	<	try:
135	<	self.pset = cfg_params['CMSSW.pset']
136	<	log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset)
134	>	if not cfg_params.has_key('CMSSW.pset'):
135	>	raise CrabException("PSet file missing. Cannot run cmsRun ")
136	>	self.pset = cfg_params['CMSSW.pset']
137	>	common.logger.log(10-1, "Cmssw::Cmssw(): PSet file = "+self.pset)
138	>	if self.pset.lower() != 'none' :
139		if (not os.path.exists(self.pset)):
140		raise CrabException("User defined PSet file "+self.pset+" does not exist")
141	<	except KeyError:
142	<	raise CrabException("PSet file missing. Cannot run cmsRun ")
141	>	else:
142	>	self.pset = None
143
144		# output files
145	<	try:
146	<	self.output_file = []
145	>	## stuff which must be returned always via sandbox
146	>	self.output_file_sandbox = []
147
148	<	tmp = cfg_params['CMSSW.output_file']
149	<	if tmp != '':
96	<	tmpOutFiles = string.split(cfg_params['CMSSW.output_file'],',')
97	<	log.debug(7, 'cmssw::cmssw(): output files '+str(tmpOutFiles))
98	<	for tmp in tmpOutFiles:
99	<	tmp=string.strip(tmp)
100	<	self.output_file.append(tmp)
101	<	pass
102	<	else:
103	<	log.message("No output file defined: only stdout/err will be available")
104	<	pass
105	<	pass
106	<	except KeyError:
107	<	log.message("No output file defined: only stdout/err will be available")
108	<	pass
148	>	# add fjr report by default via sandbox
149	>	self.output_file_sandbox.append(self.fjrFileName)
150
151	<	# script_exe file as additional file in inputSandbox
152	<	try:
153	<	self.scriptExe = cfg_params['USER.script_exe']
154	<	self.additional_inbox_files.append(self.scriptExe)
155	<	if self.scriptExe != '':
156	<	if not os.path.isfile(self.scriptExe):
157	<	msg ="WARNING. file "+self.scriptExe+" not found"
158	<	raise CrabException(msg)
159	<	except KeyError:
160	<	pass
161	<
162	<	## additional input files
163	<	try:
164	<	tmpAddFiles = string.split(cfg_params['CMSSW.additional_input_files'],',')
124	<	for tmp in tmpAddFiles:
125	<	if not os.path.exists(tmp):
126	<	raise CrabException("Additional input file not found: "+tmp)
127	<	tmp=string.strip(tmp)
128	<	self.additional_inbox_files.append(tmp)
129	<	pass
130	<	pass
131	<	except KeyError:
132	<	pass
133	<
134	<	# files per job
135	<	try:
136	<	self.filesPerJob = int(cfg_params['CMSSW.files_per_jobs']) #Daniele
137	<	self.selectFilesPerJob = 1
138	<	except KeyError:
139	<	self.filesPerJob = 0
140	<	self.selectFilesPerJob = 0
151	>	# other output files to be returned via sandbox or copied to SE
152	>	outfileflag = False
153	>	self.output_file = []
154	>	tmp = cfg_params.get('CMSSW.output_file',None)
155	>	if tmp :
156	>	self.output_file = [x.strip() for x in tmp.split(',')]
157	>	outfileflag = True #output found
158	>
159	>	self.scriptExe = cfg_params.get('USER.script_exe',None)
160	>	if self.scriptExe :
161	>	if not os.path.isfile(self.scriptExe):
162	>	msg ="ERROR. file "+self.scriptExe+" not found"
163	>	raise CrabException(msg)
164	>	self.additional_inbox_files.append(string.strip(self.scriptExe))
165
166	<	## Events per job
167	<	try:
144	<	self.eventsPerJob =int( cfg_params['CMSSW.events_per_job'])
145	<	self.selectEventsPerJob = 1
146	<	except KeyError:
147	<	self.eventsPerJob = -1
148	<	self.selectEventsPerJob = 0
149	<
150	<	# To be implemented
151	<	# ## number of jobs
152	<	# try:
153	<	#     self.numberOfJobs =int( cfg_params['CMSSW.number_of_job'])
154	<	#     self.selectNumberOfJobs = 1
155	<	# except KeyError:
156	<	#     self.selectNumberOfJobs = 0
166	>	self.AdditionalArgs = cfg_params.get('USER.script_arguments',None)
167	>	if self.AdditionalArgs : self.AdditionalArgs = string.replace(self.AdditionalArgs,',',' ')
168
169	<	if (self.selectFilesPerJob == self.selectEventsPerJob):
170	<	msg = 'Must define either files_per_jobs or events_per_job'
169	>	if self.datasetPath == None and self.pset == None and self.scriptExe == '' :
170	>	msg ="Error. script_exe  not defined"
171		raise CrabException(msg)
172
173	<	if (self.selectEventsPerJob  and not self.datasetPath == None):
174	<	msg = 'Splitting according to events_per_job available only with None as datasetpath'
164	<	raise CrabException(msg)
165	<
166	<	try:
167	<	self.total_number_of_events = int(cfg_params['CMSSW.total_number_of_events'])
168	<	except KeyError:
169	<	msg = 'Must define total_number_of_events'
170	<	raise CrabException(msg)
171	<
172	<	CEBlackList = []
173	<	try:
174	<	tmpBad = string.split(cfg_params['EDG.ce_black_list'],',')
175	<	for tmp in tmpBad:
176	<	tmp=string.strip(tmp)
177	<	CEBlackList.append(tmp)
178	<	except KeyError:
179	<	pass
180	<
181	<	self.reCEBlackList=[]
182	<	for bad in CEBlackList:
183	<	self.reCEBlackList.append(re.compile( bad ))
184	<
185	<	common.logger.debug(5,'CEBlackList: '+str(CEBlackList))
173	>	# use parent files...
174	>	self.useParent = int(self.cfg_params.get('CMSSW.use_parent',0))
175
176	<	CEWhiteList = []
177	<	try:
178	<	tmpGood = string.split(cfg_params['EDG.ce_white_list'],',')
179	<	for tmp in tmpGood:
180	<	tmp=string.strip(tmp)
181	<	CEWhiteList.append(tmp)
182	<	except KeyError:
176	>	## additional input files
177	>	if cfg_params.has_key('USER.additional_input_files'):
178	>	tmpAddFiles = string.split(cfg_params['USER.additional_input_files'],',')
179	>	for tmp in tmpAddFiles:
180	>	tmp = string.strip(tmp)
181	>	dirname = ''
182	>	if not tmp[0]=="/": dirname = "."
183	>	files = []
184	>	if string.find(tmp,"*")>-1:
185	>	files = glob.glob(os.path.join(dirname, tmp))
186	>	if len(files)==0:
187	>	raise CrabException("No additional input file found with this pattern: "+tmp)
188	>	else:
189	>	files.append(tmp)
190	>	for file in files:
191	>	if not os.path.exists(file):
192	>	raise CrabException("Additional input file not found: "+file)
193	>	pass
194	>	self.additional_inbox_files.append(string.strip(file))
195	>	pass
196		pass
197	+	common.logger.debug("Additional input files: "+str(self.additional_inbox_files))
198	+	pass
199
196	–	#print 'CEWhiteList: ',CEWhiteList
197	–	self.reCEWhiteList=[]
198	–	for Good in CEWhiteList:
199	–	self.reCEWhiteList.append(re.compile( Good ))
200
201	<	common.logger.debug(5,'CEWhiteList: '+str(CEWhiteList))
202	<
203	<	self.PsetEdit = PsetManipulator.PsetManipulator(self.pset) #Daniele Pset
201	>	## New method of dealing with seeds
202	>	self.incrementSeeds = []
203	>	self.preserveSeeds = []
204	>	if cfg_params.has_key('CMSSW.preserve_seeds'):
205	>	tmpList = cfg_params['CMSSW.preserve_seeds'].split(',')
206	>	for tmp in tmpList:
207	>	tmp.strip()
208	>	self.preserveSeeds.append(tmp)
209	>	if cfg_params.has_key('CMSSW.increment_seeds'):
210	>	tmpList = cfg_params['CMSSW.increment_seeds'].split(',')
211	>	for tmp in tmpList:
212	>	tmp.strip()
213	>	self.incrementSeeds.append(tmp)
214	>
215	>	# Copy/return/publish
216	>	self.copy_data = int(cfg_params.get('USER.copy_data',0))
217	>	self.return_data = int(cfg_params.get('USER.return_data',0))
218	>	self.publish_data = int(cfg_params.get('USER.publish_data',0))
219	>	if (self.publish_data == 1):
220	>	if not cfg_params.has_key('USER.publish_data_name'):
221	>	raise CrabException('Cannot publish output data, because you did not specify USER.publish_data_name parameter in the crab.cfg file')
222	>	else:
223	>	self.processedDataset = cfg_params['USER.publish_data_name']
224
225	+	self.conf = {}
226	+	self.conf['pubdata'] = None
227	+	# number of jobs requested to be created, limit obj splitting DD
228		#DBSDLS-start
229	<	## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
229	>	## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
230		self.maxEvents=0  # max events available   ( --> check the requested nb. of evts in Creator.py)
231		self.DBSPaths={}  # all dbs paths requested ( --> input to the site local discovery script)
232	+	self.jobDestination=[]  # Site destination(s) for each job (list of lists)
233		## Perform the data location and discovery (based on DBS/DLS)
234		## SL: Don't if NONE is specified as input (pythia use case)
235	<	common.analisys_common_info['sites']=None
236	<	if self.datasetPath:
237	<	self.DataDiscoveryAndLocation(cfg_params)
238	<	#DBSDLS-end
235	>	blockSites = {}
236	>	#wmbs
237	>	self.automation = int(self.cfg_params.get('WMBS.automation',0))
238	>	if self.automation == 0:
239	>	if self.datasetPath:
240	>	blockSites = self.DataDiscoveryAndLocation(cfg_params)
241	>	#DBSDLS-end
242	>	self.conf['blockSites']=blockSites
243	>
244	>	## Select Splitting
245	>	splitByRun = int(cfg_params.get('CMSSW.split_by_run',0))
246	>
247	>	if self.selectNoInput:
248	>	if self.pset == None:
249	>	self.algo = 'ForScript'
250	>	else:
251	>	self.algo = 'NoInput'
252	>	self.conf['managedGenerators']=self.managedGenerators
253	>	self.conf['generator']=self.generator
254	>	elif self.ads or self.lumiMask or self.lumiParams:
255	>	self.algo = 'LumiBased'
256	>	if splitByRun:
257	>	msg = "Cannot combine split by run with lumi_mask, ADS, " \
258	>	"or lumis_per_job. Use split by lumi mode instead."
259	>	raise CrabException(msg)
260
261	<	self.tgzNameWithPath = self.getTarBall(self.executable)
261	>	elif splitByRun ==1:
262	>	self.algo = 'RunBased'
263	>	else:
264	>	self.algo = 'EventBased'
265	>	common.logger.debug("Job splitting method: %s" % self.algo)
266
267	<	# modify Pset
268	<	if (self.datasetPath): # standard job
269	<	self.PsetEdit.maxEvent(self.eventsPerJob) #Daniele
270	<	self.PsetEdit.inputModule("INPUT") #Daniele
271	<
272	<	else:  # pythia like job
273	<	self.PsetEdit.maxEvent(self.eventsPerJob) #Daniele
274	<	self.PsetEdit.pythiaSeed("INPUT") #Daniele
275	<	try:
276	<	self.sourceSeed = int(cfg_params['CMSSW.pythia_seed'])
277	<	except KeyError:
278	<	self.sourceSeed = 123456
279	<	common.logger.message("No seed given, will use "+str(self.sourceSeed))
280	<
281	<	self.PsetEdit.psetWriter(self.configFilename())
282	<
283	<	## Select Splitting
284	<	if self.selectFilesPerJob: self.jobSplittingPerFiles()
285	<	elif self.selectEventsPerJob: self.jobSplittingPerEvents()
286	<	else:
287	<	msg = 'Don\'t know how to split...'
267	>	splitter = JobSplitter(self.cfg_params,self.conf)
268	>	self.dict = splitter.Algos()[self.algo]()
269	>
270	>	self.argsFile= '%s/arguments.xml'%common.work_space.shareDir()
271	>	self.rootArgsFilename= 'arguments'
272	>	# modify Pset only the first time
273	>	if isNew:
274	>	if self.pset != None: self.ModifyPset()
275	>
276	>	## Prepare inputSandbox TarBall (only the first time)
277	>	self.tarNameWithPath = self.getTarBall(self.executable)
278	>
279	>
280	>	def ModifyPset(self):
281	>	import PsetManipulator as pp
282	>
283	>	# If pycfg_params set, fake out the config script
284	>	# to make it think it was called with those args
285	>	pycfg_params = self.cfg_params.get('CMSSW.pycfg_params',None)
286	>	if pycfg_params:
287	>	trueArgv = sys.argv
288	>	sys.argv = [self.pset]
289	>	sys.argv.extend(pycfg_params.split(' '))
290	>	PsetEdit = pp.PsetManipulator(self.pset)
291	>	if pycfg_params: # Restore original sys.argv
292	>	sys.argv = trueArgv
293	>
294	>	try:
295	>	# Add FrameworkJobReport to parameter-set, set max events.
296	>	# Reset later for data jobs by writeCFG which does all modifications
297	>	PsetEdit.maxEvent(1)
298	>	PsetEdit.skipEvent(0)
299	>	PsetEdit.psetWriter(self.configFilename())
300	>	## If present, add TFileService to output files
301	>	if not int(self.cfg_params.get('CMSSW.skip_tfileservice_output',0)):
302	>	tfsOutput = PsetEdit.getTFileService()
303	>	if tfsOutput:
304	>	if tfsOutput in self.output_file:
305	>	common.logger.debug("Output from TFileService "+tfsOutput+" already in output files")
306	>	else:
307	>	outfileflag = True #output found
308	>	self.output_file.append(tfsOutput)
309	>	common.logger.info("Adding "+tfsOutput+" (from TFileService) to list of output files")
310	>	pass
311	>	pass
312	>
313	>	# If requested, add PoolOutputModule to output files
314	>	### FEDE FOR MULTI ###
315	>	#edmOutput = PsetEdit.getPoolOutputModule()
316	>	edmOutputDict = PsetEdit.getPoolOutputModule()
317	>	common.logger.debug("(test) edmOutputDict = "+str(edmOutputDict))
318	>	filter_dict = {}
319	>	for key in edmOutputDict.keys():
320	>	filter_dict[key]=edmOutputDict[key]['dataset']
321	>	common.logger.debug("(test) filter_dict for multi =  "+str(filter_dict))
322	>
323	>	#### in CMSSW.sh: export var_filter
324	>
325	>	self.var_filter = json.dumps(filter_dict)
326	>	common.logger.debug("(test) var_filter for multi =  "+self.var_filter)
327	>
328	>	edmOutput = edmOutputDict.keys()
329	>	if int(self.cfg_params.get('CMSSW.get_edm_output',0)):
330	>	if edmOutput:
331	>	for outputFile in edmOutput:
332	>	if outputFile in self.output_file:
333	>	common.logger.debug("Output from PoolOutputModule "+outputFile+" already in output files")
334	>	else:
335	>	self.output_file.append(outputFile)
336	>	common.logger.info("Adding "+outputFile+" (from PoolOutputModule) to list of output files")
337	>	# not requested, check anyhow to avoid accidental T2 overload
338	>	else:
339	>	if edmOutput:
340	>	missedFiles = []
341	>	for outputFile in edmOutput:
342	>	if outputFile not in self.output_file:
343	>	missedFiles.append(outputFile)
344	>	if missedFiles:
345	>	msg  = "ERROR: PoolOutputModule(s) are present in your ParameteSet %s \n"%self.pset
346	>	msg += "    but the file(s) produced ( %s ) are not in the list of output files\n" % ', '.join(missedFiles)
347	>	msg += "WARNING: please remove them. If you want to keep them, add the file(s) to output_files or use CMSSW.get_edm_output = 1\n"
348	>	if int(self.cfg_params.get('CMSSW.ignore_edm_output',0)):
349	>	msg += "    CMSSW.ignore_edm_output==1 : Hope you know what you are doing...\n"
350	>	common.logger.info(msg)
351	>	else :
352	>	raise CrabException(msg)
353	>
354	>	if (PsetEdit.getBadFilesSetting()):
355	>	msg = "WARNING: You have set skipBadFiles to True. This will continue processing on some errors and you may not be notified."
356	>	common.logger.info(msg)
357	>
358	>	except CrabException, msg:
359	>	common.logger.info(str(msg))
360	>	msg='Error while manipulating ParameterSet (see previous message, if any): exiting...'
361		raise CrabException(msg)
362
363	+	valid = re.compile('^[\w\.\-]+$')
364	+	for fileName in self.output_file:
365	+	if not valid.match(fileName):
366	+	msg = "The file %s may only contain alphanumeric characters and -, _, ." % fileName
367	+	raise CrabException(msg)
368	+
369
370		def DataDiscoveryAndLocation(self, cfg_params):
371
372	<	common.logger.debug(10,"CMSSW::DataDiscoveryAndLocation()")
372	>	import DataDiscovery
373	>	import DataLocation
374	>	common.logger.log(10-1,"CMSSW::DataDiscoveryAndLocation()")
375
376		datasetPath=self.datasetPath
377
248	–	## TODO
249	–	dataTiersList = ""
250	–	dataTiers = dataTiersList.split(',')
251	–
378		## Contact the DBS
379	+	common.logger.info("Contacting Data Discovery Services ...")
380		try:
381	<	self.pubdata=DataDiscovery_EDM.DataDiscovery_EDM(datasetPath, dataTiers, cfg_params)
381	>	self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params,self.skip_blocks)
382		self.pubdata.fetchDBSInfo()
383
384	<	except DataDiscovery_EDM.NotExistingDatasetError, ex :
384	>	except DataDiscovery.NotExistingDatasetError, ex :
385		msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
386		raise CrabException(msg)
387	<
261	<	except DataDiscovery_EDM.NoDataTierinProvenanceError, ex :
387	>	except DataDiscovery.NoDataTierinProvenanceError, ex :
388		msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
389		raise CrabException(msg)
390	<	except DataDiscovery_EDM.DataDiscoveryError, ex:
391	<	msg = 'ERROR ***: failed Data Discovery in DBS  %s'%ex.getErrorMessage()
390	>	except DataDiscovery.DataDiscoveryError, ex:
391	>	msg = 'ERROR ***: failed Data Discovery in DBS :  %s'%ex.getErrorMessage()
392		raise CrabException(msg)
393
394	<	## get list of all required data in the form of dbs paths  (dbs path = /dataset/datatier/owner)
395	<	## self.DBSPaths=self.pubdata.getDBSPaths()
270	<	common.logger.message("Required data are :"+self.datasetPath)
271	<
272	<	filesbyblock=self.pubdata.getFiles()
273	<	self.AllInputFiles=filesbyblock.values()
274	<	self.files = self.AllInputFiles
275	<
276	<	## TEMP
277	<	#    self.filesTmp = filesbyblock.values()
278	<	#    self.files = []
279	<	#    locPath='rfio:cmsbose2.bo.infn.it:/flatfiles/SE00/cms/fanfani/ProdTest/'
280	<	#    locPath=''
281	<	#    tmp = []
282	<	#    for file in self.filesTmp[0]:
283	<	#        tmp.append(locPath+file)
284	<	#    self.files.append(tmp)
285	<	## END TEMP
394	>	self.filesbyblock=self.pubdata.getFiles()
395	>	self.conf['pubdata']=self.pubdata
396
397		## get max number of events
398	<	#common.logger.debug(10,"number of events for primary fileblocks %i"%self.pubdata.getMaxEvents())
289	<	self.maxEvents=self.pubdata.getMaxEvents() ##  self.maxEvents used in Creator.py
290	<	common.logger.message("\nThe number of available events is %s"%self.maxEvents)
398	>	self.maxEvents=self.pubdata.getMaxEvents()
399
400		## Contact the DLS and build a list of sites hosting the fileblocks
401		try:
402	<	dataloc=DataLocation_EDM.DataLocation_EDM(filesbyblock.keys(),cfg_params)
402	>	dataloc=DataLocation.DataLocation(self.filesbyblock.keys(),cfg_params)
403		dataloc.fetchDLSInfo()
296	–	except DataLocation_EDM.DataLocationError , ex:
297	–	msg = 'ERROR ***: failed Data Location in DLS \n %s '%ex.getErrorMessage()
298	–	raise CrabException(msg)
299	–
300	–	allsites=dataloc.getSites()
301	–	common.logger.debug(5,"sites are %s"%allsites)
302	–	sites=self.checkBlackList(allsites)
303	–	common.logger.debug(5,"sites are (after black list) %s"%sites)
304	–	sites=self.checkWhiteList(sites)
305	–	common.logger.debug(5,"sites are (after white list) %s"%sites)
404
405	<	if len(sites)==0:
406	<	msg = 'No sites hosting all the needed data! Exiting... '
405	>	except DataLocation.DataLocationError , ex:
406	>	msg = 'ERROR ***: failed Data Location in DLS \n %s '%ex.getErrorMessage()
407		raise CrabException(msg)
408
311	–	common.logger.message("List of Sites hosting the data : "+str(sites))
312	–	common.logger.debug(6, "List of Sites: "+str(sites))
313	–	common.analisys_common_info['sites']=sites    ## used in SchedulerEdg.py in createSchScript
314	–	self.setParam_('TargetCE', ','.join(sites))
315	–	return
316	–
317	–	def jobSplittingPerFiles(self):
318	–	"""
319	–	Perform job splitting based on number of files to be accessed per job
320	–	"""
321	–	common.logger.debug(5,'Splitting per input files')
322	–	common.logger.message('Required '+str(self.filesPerJob)+' files per job ')
323	–	common.logger.message('Required '+str(self.total_number_of_events)+' events in total ')
324	–
325	–	## TODO: SL need to have (from DBS) a detailed list of how many events per each file
326	–	n_tot_files = (len(self.files[0]))
327	–	#print "n_tot_files = ", n_tot_files
328	–	## SL: this is wrong if the files have different number of events
329	–	#print "self.maxEvents = ", self.maxEvents
330	–	evPerFile = int(self.maxEvents)/n_tot_files
331	–	#print "evPerFile = int(self.maxEvents)/n_tot_files =  ", evPerFile
332	–
333	–	common.logger.debug(5,'Events per File '+str(evPerFile))
334	–
335	–	## if asked to process all events, do it
336	–	if self.total_number_of_events == -1:
337	–	self.total_number_of_events=self.maxEvents
338	–	self.total_number_of_jobs = int(n_tot_files)*1/int(self.filesPerJob)
339	–	common.logger.message(str(self.total_number_of_jobs)+' jobs will be created for all available events '+str(self.total_number_of_events)+' events')
340	–
341	–	else:
342	–	#print "self.total_number_of_events = ", self.total_number_of_events
343	–	#print "evPerFile = ", evPerFile
344	–	self.total_number_of_files = int(self.total_number_of_events/evPerFile)
345	–	#print "self.total_number_of_files = int(self.total_number_of_events/evPerFile) = " , self.total_number_of_files
346	–	## SL: if ask for less event than what is computed to be available on a
347	–	##     file, process the first file anyhow.
348	–	if self.total_number_of_files == 0:
349	–	self.total_number_of_files = self.total_number_of_files + 1
350	–
351	–
352	–	common.logger.debug(5,'N files  '+str(self.total_number_of_files))
353	–
354	–	check = 0
355	–
356	–	## Compute the number of jobs
357	–	#self.total_number_of_jobs = int(n_tot_files)*1/int(self.filesPerJob)
358	–	#print "self.total_number_of_files = ", self.total_number_of_files
359	–	#print "self.filesPerJob = ", self.filesPerJob
360	–	self.total_number_of_jobs = int(self.total_number_of_files/self.filesPerJob)
361	–	#print "self.total_number_of_jobs = ", self.total_number_of_jobs
362	–	common.logger.debug(5,'N jobs  '+str(self.total_number_of_jobs))
363	–
364	–	## is there any remainder?
365	–	check = int(self.total_number_of_files) - (int(self.total_number_of_jobs)*self.filesPerJob)
366	–
367	–	common.logger.debug(5,'Check  '+str(check))
368	–
369	–	if check > 0:
370	–	self.total_number_of_jobs =  self.total_number_of_jobs + 1
371	–	common.logger.message('Warning: last job will be created with '+str(check)+' files')
372	–
373	–	#common.logger.message(str(self.total_number_of_jobs)+' jobs will be created for a total of '+str((self.total_number_of_jobs-1)*self.filesPerJob*evPerFile + check*evPerFile)+' events')
374	–	common.logger.message(str(self.total_number_of_jobs)+' jobs will be created for a total of '+str((self.total_number_of_jobs)*self.filesPerJob*evPerFile + check*evPerFile)+' events')
375	–	pass
376	–
377	–	list_of_lists = []
378	–	for i in xrange(0, int(n_tot_files), self.filesPerJob):
379	–	parString = "\\{"
380	–
381	–	params = self.files[0][i: i+self.filesPerJob]
382	–	for i in range(len(params) - 1):
383	–	parString += '\\\"' + params[i] + '\\\"\,'
384	–
385	–	parString += '\\\"' + params[len(params) - 1] + '\\\"\\}'
386	–	list_of_lists.append(parString)
387	–	pass
409
410	<	self.list_of_args = list_of_lists
411	<	#print self.list_of_args
412	<	return
413	<
414	<	def jobSplittingPerEvents(self):
415	<	"""
416	<	Perform job splitting based on number of event per job
396	<	"""
397	<	common.logger.debug(5,'Splitting per events')
398	<	common.logger.message('Required '+str(self.eventsPerJob)+' events per job ')
399	<	common.logger.message('Required '+str(self.total_number_of_events)+' events in total ')
410	>	unsorted_sites = dataloc.getSites()
411	>	sites = self.filesbyblock.fromkeys(self.filesbyblock,'')
412	>	for lfn in self.filesbyblock.keys():
413	>	if unsorted_sites.has_key(lfn):
414	>	sites[lfn]=unsorted_sites[lfn]
415	>	else:
416	>	sites[lfn]=[]
417
418	<	if (self.total_number_of_events < 0):
419	<	msg='Cannot split jobs per Events with "-1" as total number of events'
418	>	if len(sites)==0:
419	>	msg = 'ERROR ***: no location for any of the blocks of this dataset: \n\t %s \n'%datasetPath
420	>	msg += "\tMaybe the dataset is located only at T1's (or at T0), where analysis jobs are not allowed\n"
421	>	msg += "\tPlease check DataDiscovery page https://cmsweb.cern.ch/dbs_discovery/\n"
422		raise CrabException(msg)
423
424	<	self.total_number_of_jobs = int(self.total_number_of_events/self.eventsPerJob)
425	<
426	<	print "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"
427	<	print "self.total_number_of_events = ", self.total_number_of_events
428	<	print "self.eventsPerJob = ", self.eventsPerJob
429	<	print "self.total_number_of_jobs = ", self.total_number_of_jobs
411	<	print "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"
412	<
413	<	common.logger.debug(5,'N jobs  '+str(self.total_number_of_jobs))
424	>	allSites = []
425	>	listSites = sites.values()
426	>	for listSite in listSites:
427	>	for oneSite in listSite:
428	>	allSites.append(oneSite)
429	>	[allSites.append(it) for it in allSites if not allSites.count(it)]
430
415	–	# is there any remainder?
416	–	check = int(self.total_number_of_events) - (int(self.total_number_of_jobs)*self.eventsPerJob)
431
432	<	common.logger.debug(5,'Check  '+str(check))
432	>	# screen output
433	>	if self.ads or self.lumiMask:
434	>	common.logger.info("Requested (A)DS %s has %s block(s)." %
435	>	(datasetPath, len(self.filesbyblock.keys())))
436	>	else:
437	>	common.logger.info("Requested dataset: " + datasetPath + \
438	>	" has " + str(self.maxEvents) + " events in " + \
439	>	str(len(self.filesbyblock.keys())) + " blocks.\n")
440
441	<	if check > 0:
421	<	common.logger.message('Warning: asked '+self.total_number_of_events+' but will do only '+(int(self.total_number_of_jobs)*self.eventsPerJob))
441	>	return sites
442
423	–	common.logger.message(str(self.total_number_of_jobs)+' jobs will be created for a total of '+str(self.total_number_of_jobs*self.eventsPerJob)+' events')
443
444	<	# argument is seed number.$i
426	<	self.list_of_args = []
427	<	for i in range(self.total_number_of_jobs):
428	<	self.list_of_args.append(int(str(self.sourceSeed)+str(i)))
429	<	print self.list_of_args
444	>	def split(self, jobParams,firstJobID):
445
446	<	return
446	>	jobParams = self.dict['args']
447	>	njobs = self.dict['njobs']
448	>	self.jobDestination = self.dict['jobDestination']
449	>
450	>	if njobs == 0:
451	>	raise CrabException("Asked to split zero jobs: aborting")
452	>	if not self.server and not self.local and njobs > 500:
453	>	raise CrabException("The CRAB client will not submit more than 500 jobs. You must use the server mode.")
454
433	–	def split(self, jobParams):
434	–
435	–	common.jobDB.load()
436	–	#### Fabio
437	–	njobs = self.total_number_of_jobs
438	–	arglist = self.list_of_args
455		# create the empty structure
456		for i in range(njobs):
457		jobParams.append("")
442	–
443	–	for job in range(njobs):
444	–	jobParams[job] = str(arglist[job])
445	–	common.jobDB.setArguments(job, jobParams[job])
458
459	<	common.jobDB.save()
459	>	listID=[]
460	>	listField=[]
461	>	listDictions=[]
462	>	exist= os.path.exists(self.argsFile)
463	>	for id in range(njobs):
464	>	job = id + int(firstJobID)
465	>	listID.append(job+1)
466	>	job_ToSave ={}
467	>	concString = ' '
468	>	argu=''
469	>	str_argu = str(job+1)
470	>	if len(jobParams[id]):
471	>	argu = {'JobID': job+1}
472	>	for i in range(len(jobParams[id])):
473	>	argu[self.dict['params'][i]]=jobParams[id][i]
474	>	if len(jobParams[id])==1: self.NumEvents = jobParams[id][i]
475	>	# just for debug
476	>	str_argu += concString.join(jobParams[id])
477	>	if argu != '': listDictions.append(argu)
478	>	job_ToSave['arguments']= '%d %d'%( (job+1), 0)
479	>	job_ToSave['dlsDestination']= self.jobDestination[id]
480	>	listField.append(job_ToSave)
481	>	from ProdCommon.SiteDB.CmsSiteMapper import CmsSEMap
482	>	cms_se = CmsSEMap()
483	>	msg="Job  %s  Arguments:  %s\n"%(str(job+1),str_argu)
484	>	msg+="\t  Destination: %s "%(str(self.jobDestination[id]))
485	>	SEDestination = [cms_se[dest] for dest in self.jobDestination[id]]
486	>	msg+="\t  CMSDestination: %s "%(str(SEDestination))
487	>	common.logger.log(10-1,msg)
488	>	# write xml
489	>	if len(listDictions):
490	>	if exist==False: self.CreateXML()
491	>	self.addEntry(listDictions)
492	>	common._db.updateJob_(listID,listField)
493		return
449	–
450	–	def getJobTypeArguments(self, nj, sched):
451	–	return common.jobDB.arguments(nj)
452	–
453	–	def numberOfJobs(self):
454	–	# Fabio
455	–	return self.total_number_of_jobs
494
495	<	def checkBlackList(self, allSites):
496	<	if len(self.reCEBlackList)==0: return allSites
497	<	sites = []
498	<	for site in allSites:
499	<	common.logger.debug(10,'Site '+site)
500	<	good=1
463	<	for re in self.reCEBlackList:
464	<	if re.search(site):
465	<	common.logger.message('CE in black list, skipping site '+site)
466	<	good=0
467	<	pass
468	<	if good: sites.append(site)
469	<	if len(sites) == 0:
470	<	common.logger.debug(3,"No sites found after BlackList")
471	<	return sites
495	>	def CreateXML(self):
496	>	"""
497	>	"""
498	>	result = IMProvNode( self.rootArgsFilename )
499	>	outfile = file( self.argsFile, 'w').write(str(result))
500	>	return
501
502	<	def checkWhiteList(self, allSites):
502	>	def addEntry(self, listDictions):
503	>	"""
504	>	_addEntry_
505
506	<	if len(self.reCEWhiteList)==0: return allSites
507	<	sites = []
508	<	for site in allSites:
509	<	good=0
510	<	for re in self.reCEWhiteList:
511	<	if re.search(site):
512	<	common.logger.debug(5,'CE in white list, adding site '+site)
513	<	good=1
514	<	if not good: continue
515	<	sites.append(site)
516	<	if len(sites) == 0:
517	<	common.logger.message("No sites found after WhiteList\n")
506	>	add an entry to the xml file
507	>	"""
508	>	## load xml
509	>	improvDoc = loadIMProvFile(self.argsFile)
510	>	entrname= 'Job'
511	>	for dictions in listDictions:
512	>	report = IMProvNode(entrname , None, **dictions)
513	>	improvDoc.addNode(report)
514	>	outfile = file( self.argsFile, 'w').write(str(improvDoc))
515	>	return
516	>
517	>	def numberOfJobs(self):
518	>	#wmbs
519	>	if self.automation==0:
520	>	return self.dict['njobs']
521		else:
522	<	common.logger.debug(5,"Selected sites via WhiteList are "+str(sites)+"\n")
489	<	return sites
522	>	return None
523
524		def getTarBall(self, exe):
525		"""
526		Return the TarBall with lib and exe
527		"""
528	<
496	<	# if it exist, just return it
497	<	self.tgzNameWithPath = common.work_space.shareDir()+self.tgz_name
528	>	self.tgzNameWithPath = common.work_space.pathForTgz()+self.tgz_name
529		if os.path.exists(self.tgzNameWithPath):
530		return self.tgzNameWithPath
531
#	Line 507 | Line 538 | class Cmssw(JobType):
538
539		# First of all declare the user Scram area
540		swArea = self.scram.getSWArea_()
510	–	#print "swArea = ", swArea
511	–	swVersion = self.scram.getSWVersion()
512	–	#print "swVersion = ", swVersion
541		swReleaseTop = self.scram.getReleaseTop_()
542	<	#print "swReleaseTop = ", swReleaseTop
515	<
542	>
543		## check if working area is release top
544		if swReleaseTop == '' or swArea == swReleaseTop:
545	+	common.logger.debug("swArea = "+swArea+" swReleaseTop ="+swReleaseTop)
546		return
547
548	<	filesToBeTarred = []
549	<	## First find the executable
550	<	if (self.executable != ''):
551	<	exeWithPath = self.scram.findFile_(executable)
552	<	#           print exeWithPath
553	<	if ( not exeWithPath ):
554	<	raise CrabException('User executable '+executable+' not found')
555	<
556	<	## then check if it's private or not
557	<	if exeWithPath.find(swReleaseTop) == -1:
558	<	# the exe is private, so we must ship
559	<	common.logger.debug(5,"Exe "+exeWithPath+" to be tarred")
560	<	path = swArea+'/'
561	<	exe = string.replace(exeWithPath, path,'')
562	<	filesToBeTarred.append(exe)
548	>	import tarfile
549	>	try: # create tar ball
550	>	tar = tarfile.open(self.tgzNameWithPath, "w:gz")
551	>	## First find the executable
552	>	if (self.executable != ''):
553	>	exeWithPath = self.scram.findFile_(executable)
554	>	if ( not exeWithPath ):
555	>	raise CrabException('User executable '+executable+' not found')
556	>
557	>	## then check if it's private or not
558	>	if exeWithPath.find(swReleaseTop) == -1:
559	>	# the exe is private, so we must ship
560	>	common.logger.debug("Exe "+exeWithPath+" to be tarred")
561	>	path = swArea+'/'
562	>	# distinguish case when script is in user project area or given by full path somewhere else
563	>	if exeWithPath.find(path) >= 0 :
564	>	exe = string.replace(exeWithPath, path,'')
565	>	tar.add(path+exe,exe)
566	>	else :
567	>	tar.add(exeWithPath,os.path.basename(executable))
568	>	pass
569	>	else:
570	>	# the exe is from release, we'll find it on WN
571	>	pass
572	>
573	>	## Now get the libraries: only those in local working area
574	>	tar.dereference=True
575	>	libDir = 'lib'
576	>	lib = swArea+'/' +libDir
577	>	common.logger.debug("lib "+lib+" to be tarred")
578	>	if os.path.exists(lib):
579	>	tar.add(lib,libDir)
580	>
581	>	## Now check if module dir is present
582	>	moduleDir = 'module'
583	>	module = swArea + '/' + moduleDir
584	>	if os.path.isdir(module):
585	>	tar.add(module,moduleDir)
586	>	tar.dereference=False
587	>
588	>	## Now check if any data dir(s) is present
589	>	self.dataExist = False
590	>	todo_list = [(i, i) for i in  os.listdir(swArea+"/src")]
591	>	while len(todo_list):
592	>	entry, name = todo_list.pop()
593	>	if name.startswith('crab_0_') or  name.startswith('.') or name == 'CVS':
594	>	continue
595	>	if os.path.isdir(swArea+"/src/"+entry):
596	>	entryPath = entry + '/'
597	>	todo_list += [(entryPath + i, i) for i in  os.listdir(swArea+"/src/"+entry)]
598	>	if name == 'data':
599	>	self.dataExist=True
600	>	common.logger.debug("data "+entry+" to be tarred")
601	>	tar.add(swArea+"/src/"+entry,"src/"+entry)
602	>	pass
603		pass
604	<	else:
605	<	# the exe is from release, we'll find it on WN
604	>
605	>	### CMSSW ParameterSet
606	>	if not self.pset is None:
607	>	cfg_file = common.work_space.jobDir()+self.configFilename()
608	>	pickleFile = common.work_space.jobDir()+self.configFilename() + '.pkl'
609	>	tar.add(cfg_file,self.configFilename())
610	>	tar.add(pickleFile,self.configFilename() + '.pkl')
611	>
612	>	try:
613	>	crab_cfg_file = common.work_space.shareDir()+'/crab.cfg'
614	>	tar.add(crab_cfg_file,'crab.cfg')
615	>	except:
616		pass
617	<
618	<	## Now get the libraries: only those in local working area
619	<	libDir = 'lib'
620	<	lib = swArea+'/' +libDir
621	<	common.logger.debug(5,"lib "+lib+" to be tarred")
622	<	if os.path.exists(lib):
623	<	filesToBeTarred.append(libDir)
624	<
625	<	## Now check if module dir is present
626	<	moduleDir = 'module'
627	<	if os.path.isdir(swArea+'/'+moduleDir):
628	<	filesToBeTarred.append(moduleDir)
629	<
630	<	## Now check if the Data dir is present
631	<	dataDir = 'src/Data/'
632	<	if os.path.isdir(swArea+'/'+dataDir):
633	<	filesToBeTarred.append(dataDir)
634	<
635	<	## Create the tar-ball
636	<	if len(filesToBeTarred)>0:
637	<	cwd = os.getcwd()
638	<	os.chdir(swArea)
639	<	tarcmd = 'tar zcvf ' + self.tgzNameWithPath + ' '
640	<	for line in filesToBeTarred:
641	<	tarcmd = tarcmd + line + ' '
642	<	cout = runCommand(tarcmd)
643	<	if not cout:
644	<	raise CrabException('Could not create tar-ball')
645	<	os.chdir(cwd)
646	<	else:
647	<	common.logger.debug(5,"No files to be to be tarred")
648	<
649	<	return
650	<
651	<	def wsSetupEnvironment(self, nj):
617	>
618	>	## Add ProdCommon dir to tar
619	>	prodcommonDir = './'
620	>	prodcommonPath = os.environ['CRABDIR'] + '/' + 'external/'
621	>	neededStuff = ['ProdCommon/__init__.py','ProdCommon/FwkJobRep', 'ProdCommon/CMSConfigTools', \
622	>	'ProdCommon/Core', 'ProdCommon/MCPayloads', 'IMProv', 'ProdCommon/Storage', \
623	>	'WMCore/__init__.py','WMCore/Algorithms']
624	>	for file in neededStuff:
625	>	tar.add(prodcommonPath+file,prodcommonDir+file)
626	>
627	>	##### ML stuff
628	>	ML_file_list=['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py']
629	>	path=os.environ['CRABDIR'] + '/python/'
630	>	for file in ML_file_list:
631	>	tar.add(path+file,file)
632	>
633	>	##### Utils
634	>	Utils_file_list=['parseCrabFjr.py','writeCfg.py', 'fillCrabFjr.py','cmscp.py']
635	>	for file in Utils_file_list:
636	>	tar.add(path+file,file)
637	>
638	>	##### AdditionalFiles
639	>	tar.dereference=True
640	>	for file in self.additional_inbox_files:
641	>	tar.add(file,string.split(file,'/')[-1])
642	>	tar.dereference=False
643	>	common.logger.log(10-1,"Files in "+self.tgzNameWithPath+" : "+str(tar.getnames()))
644	>
645	>	tar.close()
646	>	except IOError, exc:
647	>	msg = 'Could not create tar-ball %s \n'%self.tgzNameWithPath
648	>	msg += str(exc)
649	>	raise CrabException(msg)
650	>	except tarfile.TarError, exc:
651	>	msg = 'Could not create tar-ball %s \n'%self.tgzNameWithPath
652	>	msg += str(exc)
653	>	raise CrabException(msg)
654	>
655	>	tarballinfo = os.stat(self.tgzNameWithPath)
656	>	if ( tarballinfo.st_size > self.MaxTarBallSize*1024*1024 ) :
657	>	if not self.server:
658	>	msg  = 'Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' + \
659	>	str(self.MaxTarBallSize) +'MB input sandbox limit \n'
660	>	msg += '      and not supported by the direct GRID submission system.\n'
661	>	msg += '      Please use the CRAB server mode by setting server_name=<NAME> in section [CRAB] of your crab.cfg.\n'
662	>	msg += '      For further infos please see https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabServerForUsers#Server_available_for_users'
663	>	else:
664	>	msg  = 'Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' +  \
665	>	str(self.MaxTarBallSize) +'MB input sandbox limit in the server.'
666	>	raise CrabException(msg)
667	>
668	>	## create tar-ball with ML stuff
669	>
670	>	def wsSetupEnvironment(self, nj=0):
671		"""
672		Returns part of a job script which prepares
673		the execution environment for the job 'nj'.
674		"""
675	<	# Prepare JobType-independent part
579	<	txt = ''
580	<
581	<	## OLI_Daniele at this level  middleware already known
675	>	psetName = 'pset.py'
676
677	<	txt += 'if [ $middleware == LCG ]; then \n'
677	>	# Prepare JobType-independent part
678	>	txt = '\n#Written by cms_cmssw::wsSetupEnvironment\n'
679	>	txt += 'echo ">>> setup environment"\n'
680	>	txt += 'echo "set SCRAM ARCH to ' + self.executable_arch + '"\n'
681	>	txt += 'export SCRAM_ARCH=' + self.executable_arch + '\n'
682	>	txt += 'echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
683	>	txt += 'if [ $middleware == LCG ] || [ $middleware == CAF ] || [ $middleware == LSF ]; then \n'
684		txt += self.wsSetupCMSLCGEnvironment_()
685		txt += 'elif [ $middleware == OSG ]; then\n'
686	<	txt += '    time=`date -u +"%s"`\n'
687	<	txt += '    WORKING_DIR=$OSG_WN_TMP/cms_$time\n'
688	<	txt += '    echo "Creating working directory: $WORKING_DIR"\n'
689	<	txt += '    /bin/mkdir -p $WORKING_DIR\n'
690	<	txt += '    if [ ! -d $WORKING_DIR ] ;then\n'
591	<	txt += '        echo "SET_CMS_ENV 10016 ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
592	<	txt += '        echo "JOB_EXIT_STATUS = 10016"\n'
593	<	txt += '        echo "JobExitCode=10016" | tee -a $RUNTIME_AREA/$repo\n'
594	<	txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
595	<	txt += '        rm -f $RUNTIME_AREA/$repo \n'
596	<	txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
597	<	txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
598	<	txt += '        exit 1\n'
686	>	txt += '    WORKING_DIR=`/bin/mktemp  -d $OSG_WN_TMP/cms_XXXXXXXXXXXX`\n'
687	>	txt += '    if [ ! $? == 0 ] ;then\n'
688	>	txt += '        echo "ERROR ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
689	>	txt += '        job_exit_code=10016\n'
690	>	txt += '        func_exit\n'
691		txt += '    fi\n'
692	+	txt += '    echo ">>> Created working directory: $WORKING_DIR"\n'
693		txt += '\n'
694		txt += '    echo "Change to working directory: $WORKING_DIR"\n'
695		txt += '    cd $WORKING_DIR\n'
696	<	txt += self.wsSetupCMSOSGEnvironment_()
696	>	txt += '    echo ">>> current directory (WORKING_DIR): $WORKING_DIR"\n'
697	>	txt += self.wsSetupCMSOSGEnvironment_()
698	>	#Setup SGE Environment
699	>	txt += 'elif [ $middleware == SGE ]; then\n'
700	>	txt += self.wsSetupCMSLCGEnvironment_()
701	>
702	>	txt += 'elif [ $middleware == ARC ]; then\n'
703	>	txt += self.wsSetupCMSLCGEnvironment_()
704	>
705	>	#Setup PBS Environment
706	>	txt += 'elif [ $middleware == PBS ]; then\n'
707	>	txt += self.wsSetupCMSLCGEnvironment_()
708	>
709		txt += 'fi\n'
710
711		# Prepare JobType-specific part
712		scram = self.scram.commandName()
713		txt += '\n\n'
714	<	txt += 'echo "### SPECIFIC JOB SETUP ENVIRONMENT ###"\n'
714	>	txt += 'echo ">>> specific cmssw setup environment:"\n'
715	>	txt += 'echo "CMSSW_VERSION =  '+self.version+'"\n'
716		txt += scram+' project CMSSW '+self.version+'\n'
717		txt += 'status=$?\n'
718		txt += 'if [ $status != 0 ] ; then\n'
719	<	txt += '   echo "SET_EXE_ENV 10034 ==>ERROR CMSSW '+self.version+' not found on `hostname`" \n'
720	<	txt += '   echo "JOB_EXIT_STATUS = 10034"\n'
721	<	txt += '   echo "JobExitCode=10034" | tee -a $RUNTIME_AREA/$repo\n'
616	<	txt += '   dumpStatus $RUNTIME_AREA/$repo\n'
617	<	txt += '   rm -f $RUNTIME_AREA/$repo \n'
618	<	txt += '   echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
619	<	txt += '   echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
620	<	## OLI_Daniele
621	<	txt += '    if [ $middleware == OSG ]; then \n'
622	<	txt += '        echo "Remove working directory: $WORKING_DIR"\n'
623	<	txt += '        cd $RUNTIME_AREA\n'
624	<	txt += '        /bin/rm -rf $WORKING_DIR\n'
625	<	txt += '        if [ -d $WORKING_DIR ] ;then\n'
626	<	txt += '            echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
627	<	txt += '            echo "JOB_EXIT_STATUS = 10018"\n'
628	<	txt += '            echo "JobExitCode=10018" | tee -a $RUNTIME_AREA/$repo\n'
629	<	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
630	<	txt += '            rm -f $RUNTIME_AREA/$repo \n'
631	<	txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
632	<	txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
633	<	txt += '        fi\n'
634	<	txt += '    fi \n'
635	<	txt += '   exit 1 \n'
719	>	txt += '    echo "ERROR ==> CMSSW '+self.version+' not found on `hostname`" \n'
720	>	txt += '    job_exit_code=10034\n'
721	>	txt += '    func_exit\n'
722		txt += 'fi \n'
637	–	txt += 'echo "CMSSW_VERSION =  '+self.version+'"\n'
723		txt += 'cd '+self.version+'\n'
724	<	### needed grep for bug in scramv1 ###
724	>	txt += 'SOFTWARE_DIR=`pwd`; export SOFTWARE_DIR\n'
725	>	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
726		txt += 'eval `'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME`\n'
727	<
727	>	txt += 'if [ $? != 0 ] ; then\n'
728	>	txt += '    echo "ERROR ==> Problem with the command: "\n'
729	>	txt += '    echo "eval \`'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME \` at `hostname`"\n'
730	>	txt += '    job_exit_code=10034\n'
731	>	txt += '    func_exit\n'
732	>	txt += 'fi \n'
733		# Handle the arguments:
734		txt += "\n"
735	<	txt += "## number of arguments (first argument always jobnumber)\n"
735	>	txt += "## number of arguments (first argument always jobnumber, the second is the resubmission number)\n"
736		txt += "\n"
737	<	txt += "narg=$#\n"
647	<	txt += "if [ $narg -lt 2 ]\n"
737	>	txt += "if [ $nargs -lt "+str(self.argsList)+" ]\n"
738		txt += "then\n"
739	<	txt += "    echo 'SET_EXE_ENV 1 ==> ERROR Too few arguments' +$narg+ \n"
740	<	txt += '    echo "JOB_EXIT_STATUS = 50113"\n'
741	<	txt += '    echo "JobExitCode=50113" | tee -a $RUNTIME_AREA/$repo\n'
652	<	txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
653	<	txt += '    rm -f $RUNTIME_AREA/$repo \n'
654	<	txt += '    echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
655	<	txt += '    echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
656	<	## OLI_Daniele
657	<	txt += '    if [ $middleware == OSG ]; then \n'
658	<	txt += '        echo "Remove working directory: $WORKING_DIR"\n'
659	<	txt += '        cd $RUNTIME_AREA\n'
660	<	txt += '        /bin/rm -rf $WORKING_DIR\n'
661	<	txt += '        if [ -d $WORKING_DIR ] ;then\n'
662	<	txt += '            echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
663	<	txt += '            echo "JOB_EXIT_STATUS = 50114"\n'
664	<	txt += '            echo "JobExitCode=50114" | tee -a $RUNTIME_AREA/$repo\n'
665	<	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
666	<	txt += '            rm -f $RUNTIME_AREA/$repo \n'
667	<	txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
668	<	txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
669	<	txt += '        fi\n'
670	<	txt += '    fi \n'
671	<	txt += "    exit 1\n"
739	>	txt += "    echo 'ERROR ==> Too few arguments' +$nargs+ \n"
740	>	txt += '    job_exit_code=50113\n'
741	>	txt += "    func_exit\n"
742		txt += "fi\n"
743		txt += "\n"
744
745		# Prepare job-specific part
746		job = common.job_list[nj]
747	<	pset = os.path.basename(job.configFilename())
748	<	txt += '\n'
749	<	if (self.datasetPath): # standard job
680	<	txt += 'InputFiles=$2\n'
681	<	txt += 'echo "Inputfiles:<$InputFiles>"\n'
682	<	txt += 'sed "s#{\'INPUT\'}#$InputFiles#" $RUNTIME_AREA/'+pset+' > pset.cfg\n'
683	<	else:  # pythia like job
684	<	txt += 'Seed=$2\n'
685	<	txt += 'echo "Seed: <$Seed>"\n'
686	<	txt += 'sed "s#INPUT#$Seed#" $RUNTIME_AREA/'+pset+' > pset.cfg\n'
747	>	if (self.datasetPath):
748	>	txt += '\n'
749	>	txt += 'DatasetPath='+self.datasetPath+'\n'
750
751	<	if len(self.additional_inbox_files) > 0:
752	<	for file in self.additional_inbox_files:
753	<	txt += 'if [ -e $RUNTIME_AREA/'+file+' ] ; then\n'
691	<	txt += '   cp $RUNTIME_AREA/'+file+' .\n'
692	<	txt += '   chmod +x '+file+'\n'
693	<	txt += 'fi\n'
694	<	pass
751	>	txt += 'PrimaryDataset='+self.primaryDataset +'\n'
752	>	txt += 'DataTier='+self.dataTier+'\n'
753	>	txt += 'ApplicationFamily=cmsRun\n'
754
755	<	txt += 'echo "### END JOB SETUP ENVIRONMENT ###"\n\n'
755	>	else:
756	>	txt += 'DatasetPath=MCDataTier\n'
757	>	txt += 'PrimaryDataset=null\n'
758	>	txt += 'DataTier=null\n'
759	>	txt += 'ApplicationFamily=MCDataTier\n'
760	>	if self.pset != None:
761	>	pset = os.path.basename(job.configFilename())
762	>	pkl  = os.path.basename(job.configFilename()) + '.pkl'
763	>	txt += '\n'
764	>	txt += 'cp  $RUNTIME_AREA/'+pset+' .\n'
765	>	txt += 'cp  $RUNTIME_AREA/'+pkl+' .\n'
766
767	<	txt += '\n'
768	<	txt += 'echo "***** cat pset.cfg *********"\n'
769	<	txt += 'cat pset.cfg\n'
770	<	txt += 'echo "****** end pset.cfg ********"\n'
771	<	txt += '\n'
772	<	# txt += 'echo "***** cat pset1.cfg *********"\n'
773	<	# txt += 'cat pset1.cfg\n'
774	<	# txt += 'echo "****** end pset1.cfg ********"\n'
767	>	txt += 'PreserveSeeds='  + ','.join(self.preserveSeeds)  + '; export PreserveSeeds\n'
768	>	txt += 'IncrementSeeds=' + ','.join(self.incrementSeeds) + '; export IncrementSeeds\n'
769	>	txt += 'echo "PreserveSeeds: <$PreserveSeeds>"\n'
770	>	txt += 'echo "IncrementSeeds:<$IncrementSeeds>"\n'
771	>
772	>	txt += 'mv -f ' + pset + ' ' + psetName + '\n'
773	>	if self.var_filter:
774	>	#print "self.var_filter = ",self.var_filter
775	>	txt += "export var_filter="+"'"+self.var_filter+"'\n"
776	>	txt += 'echo $var_filter'
777	>	else:
778	>	txt += '\n'
779	>	if self.AdditionalArgs: txt += 'export AdditionalArgs=\"%s\"\n'%(self.AdditionalArgs)
780	>	if int(self.NumEvents) != 0: txt += 'export MaxEvents=%s\n'%str(self.NumEvents)
781		return txt
782
783	<	def wsBuildExe(self, nj):
783	>	def wsUntarSoftware(self, nj=0):
784		"""
785		Put in the script the commands to build an executable
786		or a library.
787		"""
788
789	<	txt = ""
789	>	txt = '\n#Written by cms_cmssw::wsUntarSoftware\n'
790
791		if os.path.isfile(self.tgzNameWithPath):
792	<	txt += 'echo "tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'"\n'
793	<	txt += 'tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
792	>	txt += 'echo ">>> tar xzf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+' :" \n'
793	>	if  self.debug_wrapper==1 :
794	>	txt += 'tar zxvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
795	>	txt += 'ls -Al \n'
796	>	else:
797	>	txt += 'tar zxf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
798		txt += 'untar_status=$? \n'
799		txt += 'if [ $untar_status -ne 0 ]; then \n'
800	<	txt += '   echo "SET_EXE 1 ==> ERROR Untarring .tgz file failed"\n'
801	<	txt += '   echo "JOB_EXIT_STATUS = $untar_status" \n'
802	<	txt += '   echo "JobExitCode=$untar_status" | tee -a $RUNTIME_AREA/$repo\n'
724	<	txt += '   if [ $middleware == OSG ]; then \n'
725	<	txt += '       echo "Remove working directory: $WORKING_DIR"\n'
726	<	txt += '       cd $RUNTIME_AREA\n'
727	<	txt += '       /bin/rm -rf $WORKING_DIR\n'
728	<	txt += '       if [ -d $WORKING_DIR ] ;then\n'
729	<	txt += '           echo "SET_EXE 50999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Untarring .tgz file failed"\n'
730	<	txt += '           echo "JOB_EXIT_STATUS = 50999"\n'
731	<	txt += '           echo "JobExitCode=50999" | tee -a $RUNTIME_AREA/$repo\n'
732	<	txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
733	<	txt += '           rm -f $RUNTIME_AREA/$repo \n'
734	<	txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
735	<	txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
736	<	txt += '       fi\n'
737	<	txt += '   fi \n'
738	<	txt += '   \n'
739	<	txt += '   exit 1 \n'
800	>	txt += '   echo "ERROR ==> Untarring .tgz file failed"\n'
801	>	txt += '   job_exit_code=$untar_status\n'
802	>	txt += '   func_exit\n'
803		txt += 'else \n'
804		txt += '   echo "Successful untar" \n'
805		txt += 'fi \n'
806	+	txt += '\n'
807	+	txt += 'echo ">>> Include $RUNTIME_AREA in PYTHONPATH:"\n'
808	+	txt += 'if [ -z "$PYTHONPATH" ]; then\n'
809	+	txt += '   export PYTHONPATH=$RUNTIME_AREA/\n'
810	+	txt += 'else\n'
811	+	txt += '   export PYTHONPATH=$RUNTIME_AREA/:${PYTHONPATH}\n'
812	+	txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
813	+	txt += 'fi\n'
814	+	txt += '\n'
815	+
816		pass
817	<
817	>
818		return txt
819
820	<	def modifySteeringCards(self, nj):
820	>	def wsBuildExe(self, nj=0):
821		"""
822	<	modify the card provided by the user,
823	<	writing a new card into share dir
822	>	Put in the script the commands to build an executable
823	>	or a library.
824		"""
825	<
825	>
826	>	txt = '\n#Written by cms_cmssw::wsBuildExe\n'
827	>	txt += 'echo ">>> moving CMSSW software directories in `pwd`" \n'
828	>
829	>	txt += 'rm -r lib/ module/ \n'
830	>	txt += 'mv $RUNTIME_AREA/lib/ . \n'
831	>	txt += 'mv $RUNTIME_AREA/module/ . \n'
832	>	if self.dataExist == True:
833	>	txt += 'rm -r src/ \n'
834	>	txt += 'mv $RUNTIME_AREA/src/ . \n'
835	>	if len(self.additional_inbox_files)>0:
836	>	for file in self.additional_inbox_files:
837	>	txt += 'mv $RUNTIME_AREA/'+os.path.basename(file)+' . \n'
838	>
839	>	txt += 'echo ">>> Include $RUNTIME_AREA in PYTHONPATH:"\n'
840	>	txt += 'if [ -z "$PYTHONPATH" ]; then\n'
841	>	txt += '   export PYTHONPATH=$RUNTIME_AREA/\n'
842	>	txt += 'else\n'
843	>	txt += '   export PYTHONPATH=$RUNTIME_AREA/:${PYTHONPATH}\n'
844	>	txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
845	>	txt += 'fi\n'
846	>	txt += '\n'
847	>
848	>	if self.pset != None:
849	>	psetName = 'pset.py'
850	>
851	>	txt += '\n'
852	>	if self.debug_wrapper == 1:
853	>	txt += 'echo "***** cat ' + psetName + ' *********"\n'
854	>	txt += 'cat ' + psetName + '\n'
855	>	txt += 'echo "****** end ' + psetName + ' ********"\n'
856	>	txt += '\n'
857	>	txt += 'echo "***********************" \n'
858	>	txt += 'which edmConfigHash \n'
859	>	txt += 'echo "***********************" \n'
860	>	txt += 'edmConfigHash ' + psetName + ' \n'
861	>	txt += 'PSETHASH=`edmConfigHash ' + psetName + '` \n'
862	>	txt += 'echo "PSETHASH = $PSETHASH" \n'
863	>	#### FEDE temporary fix for noEdm files #####
864	>	txt += 'if [ -z "$PSETHASH" ]; then \n'
865	>	txt += '   export PSETHASH=null\n'
866	>	txt += 'fi \n'
867	>	#############################################
868	>	txt += '\n'
869	>	return txt
870	>
871	>
872		def executableName(self):
873	<	return self.executable
873	>	if self.scriptExe:
874	>	return "sh "
875	>	else:
876	>	return self.executable
877
878		def executableArgs(self):
879	<	return " -p pset.cfg"
879	>	if self.scriptExe:
880	>	return self.scriptExe + " $NJob $AdditionalArgs"
881	>	else:
882	>	return " -j $RUNTIME_AREA/crab_fjr_$NJob.xml -p pset.py"
883
884		def inputSandbox(self, nj):
885		"""
886		Returns a list of filenames to be put in JDL input sandbox.
887		"""
888		inp_box = []
764	–	# dict added to delete duplicate from input sandbox file list
765	–	seen = {}
766	–	## code
889		if os.path.isfile(self.tgzNameWithPath):
890		inp_box.append(self.tgzNameWithPath)
891	<	## config
892	<	inp_box.append(common.job_list[nj].configFilename())
893	<	## additional input files
772	<	#for file in self.additional_inbox_files:
773	<	#    inp_box.append(common.work_space.cwdDir()+file)
891	>	if os.path.isfile(self.argsFile):
892	>	inp_box.append(self.argsFile)
893	>	inp_box.append(common.work_space.jobDir() + self.scriptName)
894		return inp_box
895
896		def outputSandbox(self, nj):
#	Line 779 | Line 899 | class Cmssw(JobType):
899		"""
900		out_box = []
901
782	–	stdout=common.job_list[nj].stdout()
783	–	stderr=common.job_list[nj].stderr()
784	–
902		## User Declared output files
903	<	for out in self.output_file:
904	<	n_out = nj + 1
905	<	out_box.append(self.numberFile_(out,str(n_out)))
903	>	for out in (self.output_file+self.output_file_sandbox):
904	>	n_out = nj + 1
905	>	out_box.append(numberFile(out,str(n_out)))
906		return out_box
790	–	return []
907
792	–	def prepareSteeringCards(self):
793	–	"""
794	–	Make initial modifications of the user's steering card file.
795	–	"""
796	–	return
908
909		def wsRenameOutput(self, nj):
910		"""
911		Returns part of a job script which renames the produced files.
912		"""
913
914	<	txt = '\n'
915	<	txt += '# directory content\n'
916	<	txt += 'ls \n'
917	<	file_list = ''
918	<	for fileWithSuffix in self.output_file:
919	<	output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
920	<	file_list=file_list+output_file_num+' '
914	>	txt = '\n#Written by cms_cmssw::wsRenameOutput\n'
915	>	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
916	>	txt += 'echo ">>> current directory content:"\n'
917	>	if self.debug_wrapper==1:
918	>	txt += 'ls -Al\n'
919	>	txt += '\n'
920	>
921	>	for fileWithSuffix in (self.output_file):
922	>	output_file_num = numberFile(fileWithSuffix, '$OutUniqueID')
923		txt += '\n'
924		txt += '# check output file\n'
925	<	txt += 'ls '+fileWithSuffix+'\n'
926	<	txt += 'exe_result=$?\n'
927	<	txt += 'if [ $exe_result -ne 0 ] ; then\n'
928	<	txt += '   echo "ERROR: No output file to manage"\n'
929	<	txt += '   echo "JOB_EXIT_STATUS = $exe_result"\n'
930	<	txt += '   echo "JobExitCode=60302" | tee -a $RUNTIME_AREA/$repo\n'
931	<	txt += '   dumpStatus $RUNTIME_AREA/$repo\n'
932	<	txt += '   rm -f $RUNTIME_AREA/$repo \n'
933	<	txt += '   echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
934	<	txt += '   echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
935	<	### OLI_DANIELE
823	<	if common.scheduler.boss_scheduler_name == 'condor_g':
925	>	txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
926	>	if (self.copy_data == 1):  # For OSG nodes, file is in $WORKING_DIR, should not be moved to $RUNTIME_AREA
927	>	txt += '    mv '+fileWithSuffix+' '+output_file_num+'\n'
928	>	txt += '    ln -s `pwd`/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
929	>	else:
930	>	txt += '    mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
931	>	txt += '    ln -s $RUNTIME_AREA/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
932	>	txt += 'else\n'
933	>	txt += '    job_exit_code=60302\n'
934	>	txt += '    echo "WARNING: Output file '+fileWithSuffix+' not found"\n'
935	>	if common.scheduler.name().upper() == 'CONDOR_G':
936		txt += '    if [ $middleware == OSG ]; then \n'
937		txt += '        echo "prepare dummy output file"\n'
938		txt += '        echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
939		txt += '    fi \n'
828	–	txt += 'else\n'
829	–	txt += '   cp '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
940		txt += 'fi\n'
941	<
942	<	txt += 'cd $RUNTIME_AREA\n'
943	<	file_list=file_list[:-1]
944	<	txt += 'file_list="'+file_list+'"\n'
945	<	### OLI_DANIELE
946	<	txt += 'if [ $middleware == OSG ]; then\n'
947	<	txt += '    cd $RUNTIME_AREA\n'
948	<	txt += '    echo "Remove working directory: $WORKING_DIR"\n'
949	<	txt += '    /bin/rm -rf $WORKING_DIR\n'
950	<	txt += '    if [ -d $WORKING_DIR ] ;then\n'
841	<	txt += '        echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
842	<	txt += '        echo "JOB_EXIT_STATUS = 60999"\n'
843	<	txt += '        echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
844	<	txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
845	<	txt += '        rm -f $RUNTIME_AREA/$repo \n'
846	<	txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
847	<	txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
848	<	txt += '    fi\n'
849	<	txt += 'fi\n'
941	>	file_list = []
942	>	for fileWithSuffix in (self.output_file):
943	>	file_list.append(numberFile('$SOFTWARE_DIR/'+fileWithSuffix, '$OutUniqueID'))
944	>
945	>	txt += 'file_list="'+string.join(file_list,',')+'"\n'
946	>	txt += '\n'
947	>	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
948	>	txt += 'echo ">>> current directory content:"\n'
949	>	if self.debug_wrapper==1:
950	>	txt += 'ls -Al\n'
951		txt += '\n'
952	+	txt += 'cd $RUNTIME_AREA\n'
953	+	txt += 'echo ">>> current directory (RUNTIME_AREA):  $RUNTIME_AREA"\n'
954		return txt
955
956	<	def numberFile_(self, file, txt):
854	<	"""
855	<	append _'txt' before last extension of a file
856	<	"""
857	<	p = string.split(file,".")
858	<	# take away last extension
859	<	name = p[0]
860	<	for x in p[1:-1]:
861	<	name=name+"."+x
862	<	# add "_txt"
863	<	if len(p)>1:
864	<	ext = p[len(p)-1]
865	<	#result = name + '_' + str(txt) + "." + ext
866	<	result = name + '_' + txt + "." + ext
867	<	else:
868	<	#result = name + '_' + str(txt)
869	<	result = name + '_' + txt
870	<
871	<	return result
872	<
873	<	def getRequirements(self):
956	>	def getRequirements(self, nj=[]):
957		"""
958	<	return job requirements to add to jdl files
958	>	return job requirements to add to jdl files
959		"""
960		req = ''
961	<	if common.analisys_common_info['sw_version']:
961	>	if self.version:
962		req='Member("VO-cms-' + \
963	<	common.analisys_common_info['sw_version'] + \
963	>	self.version + \
964		'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
965	<	if common.analisys_common_info['sites']:
966	<	if len(common.analisys_common_info['sites'])>0:
967	<	req = req + ' && ('
968	<	for i in range(len(common.analisys_common_info['sites'])):
969	<	req = req + 'other.GlueCEInfoHostName == "' \
970	<	+ common.analisys_common_info['sites'][i] + '"'
971	<	if ( i < (int(len(common.analisys_common_info['sites']) - 1)) ):
972	<	req = req + ' || '
973	<	req = req + ')'
974	<	#print "req = ", req
965	>	if self.executable_arch:
966	>	req+=' && Member("VO-cms-' + \
967	>	self.executable_arch + \
968	>	'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
969	>
970	>	req = req + ' && (other.GlueHostNetworkAdapterOutboundIP)'
971	>	if ( common.scheduler.name() in ["glite"] ):
972	>	## 25-Jun-2009 SL: patch to use Cream enabled WMS
973	>	if ( self.cfg_params.get('GRID.use_cream',None) ):
974	>	req += ' && (other.GlueCEStateStatus == "Production" || other.GlueCEStateStatus == "Special")'
975	>	else:
976	>	req += ' && other.GlueCEStateStatus == "Production" '
977	>
978		return req
979
980		def configFilename(self):
981		""" return the config filename """
982	<	return self.name()+'.cfg'
982	>	return self.name()+'.py'
983
898	–	### OLI_DANIELE
984		def wsSetupCMSOSGEnvironment_(self):
985		"""
986		Returns part of a job script which is prepares
987		the execution environment and which is common for all CMS jobs.
988		"""
989	<	txt = '\n'
990	<	txt += '   echo "### SETUP CMS OSG  ENVIRONMENT ###"\n'
991	<	txt += '   if [ -f $GRID3_APP_DIR/cmssoft/cmsset_default.sh ] ;then\n'
992	<	txt += '      # Use $GRID3_APP_DIR/cmssoft/cmsset_default.sh to setup cms software\n'
993	<	txt += '       source $GRID3_APP_DIR/cmssoft/cmsset_default.sh '+self.version+'\n'
994	<	txt += '   elif [ -f $OSG_APP/cmssoft/cmsset_default.sh ] ;then\n'
995	<	txt += '      # Use $OSG_APP/cmssoft/cmsset_default.sh to setup cms software\n'
996	<	txt += '       source $OSG_APP/cmssoft/cmsset_default.sh '+self.version+'\n'
997	<	txt += '   else\n'
998	<	txt += '       echo "SET_CMS_ENV 10020 ==> ERROR $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cmsset_default.sh file not found"\n'
999	<	txt += '       echo "JOB_EXIT_STATUS = 10020"\n'
1000	<	txt += '       echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
1001	<	txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
917	<	txt += '       rm -f $RUNTIME_AREA/$repo \n'
918	<	txt += '       echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
919	<	txt += '       echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
920	<	txt += '       exit 1\n'
921	<	txt += '\n'
922	<	txt += '       echo "Remove working directory: $WORKING_DIR"\n'
923	<	txt += '       cd $RUNTIME_AREA\n'
924	<	txt += '       /bin/rm -rf $WORKING_DIR\n'
925	<	txt += '       if [ -d $WORKING_DIR ] ;then\n'
926	<	txt += '            echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cmsset_default.sh file not found"\n'
927	<	txt += '            echo "JOB_EXIT_STATUS = 10017"\n'
928	<	txt += '            echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n'
929	<	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
930	<	txt += '            rm -f $RUNTIME_AREA/$repo \n'
931	<	txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
932	<	txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
933	<	txt += '       fi\n'
934	<	txt += '\n'
935	<	txt += '       exit 1\n'
936	<	txt += '   fi\n'
989	>	txt = '\n#Written by cms_cmssw::wsSetupCMSOSGEnvironment_\n'
990	>	txt += '    echo ">>> setup CMS OSG environment:"\n'
991	>	txt += '    echo "set SCRAM ARCH to ' + self.executable_arch + '"\n'
992	>	txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
993	>	txt += '    echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
994	>	txt += '    if [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n'
995	>	txt += '      # Use $OSG_APP/cmssoft/cms/cmsset_default.sh to setup cms software\n'
996	>	txt += '        source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
997	>	txt += '    else\n'
998	>	txt += '        echo "ERROR ==> $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
999	>	txt += '        job_exit_code=10020\n'
1000	>	txt += '        func_exit\n'
1001	>	txt += '    fi\n'
1002		txt += '\n'
1003	<	txt += '   echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1004	<	txt += '   echo " END SETUP CMS OSG  ENVIRONMENT "\n'
1003	>	txt += '    echo "==> setup cms environment ok"\n'
1004	>	txt += '    echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
1005
1006		return txt
1007	<
943	<	### OLI_DANIELE
1007	>
1008		def wsSetupCMSLCGEnvironment_(self):
1009		"""
1010		Returns part of a job script which is prepares
1011		the execution environment and which is common for all CMS jobs.
1012		"""
1013	<	txt  = '   \n'
1014	<	txt += '   echo " ### SETUP CMS LCG  ENVIRONMENT ### "\n'
1015	<	txt += '   if [ ! $VO_CMS_SW_DIR ] ;then\n'
1016	<	txt += '       echo "SET_CMS_ENV 10031 ==> ERROR CMS software dir not found on WN `hostname`"\n'
1017	<	txt += '       echo "JOB_EXIT_STATUS = 10031" \n'
1018	<	txt += '       echo "JobExitCode=10031" | tee -a $RUNTIME_AREA/$repo\n'
1019	<	txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
1020	<	txt += '       rm -f $RUNTIME_AREA/$repo \n'
1021	<	txt += '       echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1022	<	txt += '       echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1023	<	txt += '       exit 1\n'
1024	<	txt += '   else\n'
1025	<	txt += '       echo "Sourcing environment... "\n'
1026	<	txt += '       if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
1027	<	txt += '           echo "SET_CMS_ENV 10020 ==> ERROR cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1028	<	txt += '           echo "JOB_EXIT_STATUS = 10020"\n'
1029	<	txt += '           echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
1030	<	txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1031	<	txt += '           rm -f $RUNTIME_AREA/$repo \n'
1032	<	txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1033	<	txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1034	<	txt += '           exit 1\n'
1035	<	txt += '       fi\n'
1036	<	txt += '       echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1037	<	txt += '       source $VO_CMS_SW_DIR/cmsset_default.sh\n'
1038	<	txt += '       result=$?\n'
1039	<	txt += '       if [ $result -ne 0 ]; then\n'
1040	<	txt += '           echo "SET_CMS_ENV 10032 ==> ERROR problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1041	<	txt += '           echo "JOB_EXIT_STATUS = 10032"\n'
1042	<	txt += '           echo "JobExitCode=10032" | tee -a $RUNTIME_AREA/$repo\n'
1043	<	txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1044	<	txt += '           rm -f $RUNTIME_AREA/$repo \n'
1045	<	txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1046	<	txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1047	<	txt += '           exit 1\n'
1048	<	txt += '       fi\n'
1049	<	txt += '   fi\n'
1050	<	txt += '   \n'
1051	<	txt += '   string=`cat /etc/redhat-release`\n'
1052	<	txt += '   echo $string\n'
1053	<	txt += '   if [[ $string = *alhalla* ]]; then\n'
1054	<	txt += '       echo "SCRAM_ARCH= $SCRAM_ARCH"\n'
1055	<	txt += '   elif [[ $string = *Enterprise* ]] || [[ $string = *cientific* ]]; then\n'
1056	<	txt += '       export SCRAM_ARCH=slc3_ia32_gcc323\n'
1057	<	txt += '       echo "SCRAM_ARCH= $SCRAM_ARCH"\n'
1058	<	txt += '   else\n'
1059	<	txt += '       echo "SET_CMS_ENV 10033 ==> ERROR OS unknown, LCG environment not initialized"\n'
1060	<	txt += '       echo "JOB_EXIT_STATUS = 10033"\n'
1061	<	txt += '       echo "JobExitCode=10033" | tee -a $RUNTIME_AREA/$repo\n'
1062	<	txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
1063	<	txt += '       rm -f $RUNTIME_AREA/$repo \n'
1064	<	txt += '       echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1065	<	txt += '       echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1066	<	txt += '       exit 1\n'
1067	<	txt += '   fi\n'
1068	<	txt += '   echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1069	<	txt += '   echo "### END SETUP CMS LCG ENVIRONMENT ###"\n'
1013	>	txt = '\n#Written by cms_cmssw::wsSetupCMSLCGEnvironment_\n'
1014	>	txt += '    echo ">>> setup CMS LCG environment:"\n'
1015	>	txt += '    echo "set SCRAM ARCH and BUILD_ARCH to ' + self.executable_arch + ' ###"\n'
1016	>	txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
1017	>	txt += '    export BUILD_ARCH='+self.executable_arch+'\n'
1018	>	txt += '    if [ ! $VO_CMS_SW_DIR ] ;then\n'
1019	>	txt += '        echo "ERROR ==> CMS software dir not found on WN `hostname`"\n'
1020	>	txt += '        job_exit_code=10031\n'
1021	>	txt += '        func_exit\n'
1022	>	txt += '    else\n'
1023	>	txt += '        echo "Sourcing environment... "\n'
1024	>	txt += '        if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
1025	>	txt += '            echo "ERROR ==> cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1026	>	txt += '            job_exit_code=10020\n'
1027	>	txt += '            func_exit\n'
1028	>	txt += '        fi\n'
1029	>	txt += '        echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1030	>	txt += '        source $VO_CMS_SW_DIR/cmsset_default.sh\n'
1031	>	txt += '        result=$?\n'
1032	>	txt += '        if [ $result -ne 0 ]; then\n'
1033	>	txt += '            echo "ERROR ==> problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1034	>	txt += '            job_exit_code=10032\n'
1035	>	txt += '            func_exit\n'
1036	>	txt += '        fi\n'
1037	>	txt += '    fi\n'
1038	>	txt += '    \n'
1039	>	txt += '    echo "==> setup cms environment ok"\n'
1040	>	return txt
1041	>
1042	>	def wsModifyReport(self, nj):
1043	>	"""
1044	>	insert the part of the script that modifies the FrameworkJob Report
1045	>	"""
1046	>
1047	>	txt = ''
1048	>	if (self.copy_data == 1):
1049	>	txt = '\n#Written by cms_cmssw::wsModifyReport\n'
1050	>
1051	>	txt += 'echo ">>> Modify Job Report:" \n'
1052	>	txt += 'chmod a+x $RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py\n'
1053	>	txt += 'echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n'
1054	>
1055	>	args = 'fjr $RUNTIME_AREA/crab_fjr_$NJob.xml json $RUNTIME_AREA/resultCopyFile n_job $OutUniqueID PrimaryDataset $PrimaryDataset  ApplicationFamily $ApplicationFamily ApplicationName $executable cmssw_version $CMSSW_VERSION psethash $PSETHASH'
1056	>
1057	>	if (self.publish_data == 1):
1058	>	txt += 'ProcessedDataset='+self.processedDataset+'\n'
1059	>	txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1060	>	args += ' UserProcessedDataset $USER-$ProcessedDataset-$PSETHASH'
1061	>
1062	>	txt += 'echo "$RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py '+str(args)+'"\n'
1063	>	txt += '$RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py '+str(args)+'\n'
1064	>	txt += 'modifyReport_result=$?\n'
1065	>	txt += 'if [ $modifyReport_result -ne 0 ]; then\n'
1066	>	txt += '    modifyReport_result=70500\n'
1067	>	txt += '    job_exit_code=$modifyReport_result\n'
1068	>	txt += '    echo "ModifyReportResult=$modifyReport_result" | tee -a $RUNTIME_AREA/$repo\n'
1069	>	txt += '    echo "WARNING: Problem with ModifyJobReport"\n'
1070	>	txt += 'else\n'
1071	>	txt += '    mv NewFrameworkJobReport.xml $RUNTIME_AREA/crab_fjr_$NJob.xml\n'
1072	>	txt += 'fi\n'
1073	>	return txt
1074	>
1075	>	def wsParseFJR(self):
1076	>	"""
1077	>	Parse the FrameworkJobReport to obtain useful infos
1078	>	"""
1079	>	txt = '\n#Written by cms_cmssw::wsParseFJR\n'
1080	>	txt += 'echo ">>> Parse FrameworkJobReport crab_fjr.xml"\n'
1081	>	txt += 'if [ -s $RUNTIME_AREA/crab_fjr_$NJob.xml ]; then\n'
1082	>	txt += '    if [ -s $RUNTIME_AREA/parseCrabFjr.py ]; then\n'
1083	>	txt += '        cmd_out=`python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --dashboard $MonitorID,$MonitorJobID '+self.debugWrap+'`\n'
1084	>	if self.debug_wrapper==1 :
1085	>	txt += '        echo "Result of parsing the FrameworkJobReport crab_fjr.xml: $cmd_out"\n'
1086	>	txt += '        executable_exit_status=`python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --exitcode`\n'
1087	>	txt += '        if [ $executable_exit_status -eq 50115 ];then\n'
1088	>	txt += '            echo ">>> crab_fjr.xml contents: "\n'
1089	>	txt += '            cat $RUNTIME_AREA/crab_fjr_$NJob.xml\n'
1090	>	txt += '            echo "Wrong FrameworkJobReport --> does not contain useful info. ExitStatus: $executable_exit_status"\n'
1091	>	txt += '        elif [ $executable_exit_status -eq -999 ];then\n'
1092	>	txt += '            echo "ExitStatus from FrameworkJobReport not available. not available. Using exit code of executable from command line."\n'
1093	>	txt += '        else\n'
1094	>	txt += '            echo "Extracted ExitStatus from FrameworkJobReport parsing output: $executable_exit_status"\n'
1095	>	txt += '        fi\n'
1096	>	txt += '    else\n'
1097	>	txt += '        echo "CRAB python script to parse CRAB FrameworkJobReport crab_fjr.xml is not available, using exit code of executable from command line."\n'
1098	>	txt += '    fi\n'
1099	>	#### Patch to check input data reading for CMSSW16x Hopefully we-ll remove it asap
1100	>	txt += '    if [ $executable_exit_status -eq 0 ];then\n'
1101	>	txt += '        echo ">>> Executable succeded  $executable_exit_status"\n'
1102	>	txt += '    fi\n'
1103	>	txt += 'else\n'
1104	>	txt += '    echo "CRAB FrameworkJobReport crab_fjr.xml is not available, using exit code of executable from command line."\n'
1105	>	txt += 'fi\n'
1106	>	txt += '\n'
1107	>	txt += 'if [ $executable_exit_status -ne 0 ];then\n'
1108	>	txt += '    echo ">>> Executable failed  $executable_exit_status"\n'
1109	>	txt += '    echo "ExeExitCode=$executable_exit_status" | tee -a $RUNTIME_AREA/$repo\n'
1110	>	txt += '    echo "EXECUTABLE_EXIT_STATUS = $executable_exit_status"\n'
1111	>	txt += '    job_exit_code=$executable_exit_status\n'
1112	>	txt += '    func_exit\n'
1113	>	txt += 'fi\n\n'
1114	>	txt += 'echo "ExeExitCode=$executable_exit_status" | tee -a $RUNTIME_AREA/$repo\n'
1115	>	txt += 'echo "EXECUTABLE_EXIT_STATUS = $executable_exit_status"\n'
1116	>	txt += 'job_exit_code=$executable_exit_status\n'
1117	>
1118		return txt
1119
1120		def setParam_(self, param, value):
#	Line 1011 | Line 1123 | class Cmssw(JobType):
1123		def getParams(self):
1124		return self._params
1125
1126	<	def setTaskid_(self):
1127	<	self._taskId = self.cfg_params['taskId']
1128	<
1129	<	def getTaskid(self):
1130	<	return self._taskId
1126	>	def outList(self,list=False):
1127	>	"""
1128	>	check the dimension of the output files
1129	>	"""
1130	>	txt = ''
1131	>	txt += 'echo ">>> list of expected files on output sandbox"\n'
1132	>	listOutFiles = []
1133	>	stdout = 'CMSSW_$NJob.stdout'
1134	>	stderr = 'CMSSW_$NJob.stderr'
1135	>	if len(self.output_file) <= 0:
1136	>	msg ="WARNING: no output files name have been defined!!\n"
1137	>	msg+="\tno output files will be reported back/staged\n"
1138	>	common.logger.info(msg)
1139	>
1140	>	if (self.return_data == 1):
1141	>	for file in (self.output_file):
1142	>	listOutFiles.append(numberFile(file, '$OutUniqueID'))
1143	>	for file in (self.output_file_sandbox):
1144	>	listOutFiles.append(numberFile(file, '$NJob'))
1145	>	listOutFiles.append(stdout)
1146	>	listOutFiles.append(stderr)
1147	>
1148	>	txt += 'echo "output files: '+string.join(listOutFiles,' ')+'"\n'
1149	>	txt += 'filesToCheck="'+string.join(listOutFiles,' ')+'"\n'
1150	>	txt += 'export filesToCheck\n'
1151	>	taskinfo={}
1152	>	taskinfo['outfileBasename'] = self.output_file
1153	>	common._db.updateTask_(taskinfo)
1154	>
1155	>	if list : return self.output_file
1156	>	return txt
1157	>
1158	>	def checkCMSSWVersion(self, url = "https://cmstags.cern.ch/cgi-bin/CmsTC/", fileName = "ReleasesXML"):
1159	>	"""
1160	>	compare current CMSSW release and arch with allowed releases
1161	>	"""
1162	>
1163	>	downloader = Downloader(url)
1164	>	goodRelease = False
1165	>
1166	>	try:
1167	>	result = downloader.config(fileName)
1168	>	except:
1169	>	common.logger.info("ERROR: Problem reading file of allowed CMSSW releases.")
1170	>
1171	>	try:
1172	>	events = pulldom.parseString(result)
1173	>
1174	>	arch     = None
1175	>	release  = None
1176	>	relType  = None
1177	>	relState = None
1178	>	for (event, node) in events:
1179	>	if event == pulldom.START_ELEMENT:
1180	>	if node.tagName == 'architecture':
1181	>	arch = node.attributes.getNamedItem('name').nodeValue
1182	>	if node.tagName == 'project':
1183	>	relType = node.attributes.getNamedItem('type').nodeValue
1184	>	relState = node.attributes.getNamedItem('state').nodeValue
1185	>	if relType == 'Production' and relState == 'Announced':
1186	>	release = node.attributes.getNamedItem('label').nodeValue
1187	>	if self.executable_arch == arch and self.version == release:
1188	>	goodRelease = True
1189	>	return goodRelease
1190	>
1191	>	if not goodRelease:
1192	>	msg = "WARNING: %s on %s is not a supported release. " % \
1193	>	(self.version, self.executable_arch)
1194	>	msg += "Submission may fail."
1195	>	common.logger.info(msg)
1196	>	except:
1197	>	common.logger.info("Problems parsing file of allowed CMSSW releases.")
1198	>
1199	>	return goodRelease
1200	>

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines