ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/cms_cmssw.py

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.49 by slacapra, Thu Oct 5 15:32:20 2006 UTC vs.
Revision 1.75 by gutsche, Sun Apr 8 23:50:33 2007 UTC

#	Line 2 | Line 2 | from JobType import JobType
2		from crab_logger import Logger
3		from crab_exceptions import *
4		from crab_util import *
5	–	import math
5		import common
6		import PsetManipulator
8	–
9	–	import DBSInfo
7		import DataDiscovery
8	+	import DataDiscovery_DBS2
9		import DataLocation
10		import Scram
11
12	<	import glob, os, string, re
12	>	import os, string, re, shutil, glob
13
14		class Cmssw(JobType):
15		def __init__(self, cfg_params, ncjobs):
#	Line 22 | Line 20 | class Cmssw(JobType):
20		self._params = {}
21		self.cfg_params = cfg_params
22
23	+	try:
24	+	self.MaxTarBallSize = float(self.cfg_params['EDG.maxtarballsize'])
25	+	except KeyError:
26	+	self.MaxTarBallSize = 100.0
27	+
28		# number of jobs requested to be created, limit obj splitting
29		self.ncjobs = ncjobs
30
31		log = common.logger
32
33		self.scram = Scram.Scram(cfg_params)
31	–	scramArea = ''
34		self.additional_inbox_files = []
35		self.scriptExe = ''
36		self.executable = ''
37	+	self.executable_arch = self.scram.getArch()
38		self.tgz_name = 'default.tgz'
39	+	self.scriptName = 'CMSSW.sh'
40		self.pset = ''      #scrip use case Da
41		self.datasetPath = '' #scrip use case Da
42
43	+	# set FJR file name
44	+	self.fjrFileName = 'crab_fjr.xml'
45	+
46		self.version = self.scram.getSWVersion()
47	+	common.taskDB.setDict('codeVersion',self.version)
48		self.setParam_('application', self.version)
49
50		### collect Data cards
51	+
52	+	## get DBS mode
53	+	try:
54	+	self.use_dbs_2 = int(self.cfg_params['CMSSW.use_dbs_2'])
55	+	except KeyError:
56	+	self.use_dbs_2 = 0
57	+
58		try:
59		tmp =  cfg_params['CMSSW.datasetpath']
60		log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
#	Line 60 | Line 75 | class Cmssw(JobType):
75		self.setParam_('owner', 'None')
76		else:
77		datasetpath_split = self.datasetPath.split("/")
78	<	self.setParam_('dataset', datasetpath_split[1])
79	<	self.setParam_('owner', datasetpath_split[-1])
78	>	if self.use_dbs_2 == 1 :
79	>	self.setParam_('dataset', datasetpath_split[1])
80	>	self.setParam_('owner', datasetpath_split[2])
81	>	else :
82	>	self.setParam_('dataset', datasetpath_split[1])
83	>	self.setParam_('owner', datasetpath_split[-1])
84
85		self.setTaskid_()
86		self.setParam_('taskId', self.cfg_params['taskId'])
#	Line 94 | Line 113 | class Cmssw(JobType):
113		raise CrabException("PSet file missing. Cannot run cmsRun ")
114
115		# output files
116	+	## stuff which must be returned always via sandbox
117	+	self.output_file_sandbox = []
118	+
119	+	# add fjr report by default via sandbox
120	+	self.output_file_sandbox.append(self.fjrFileName)
121	+
122	+	# other output files to be returned via sandbox or copied to SE
123		try:
124		self.output_file = []
99	–
125		tmp = cfg_params['CMSSW.output_file']
126		if tmp != '':
127		tmpOutFiles = string.split(cfg_params['CMSSW.output_file'],',')
#	Line 106 | Line 131 | class Cmssw(JobType):
131		self.output_file.append(tmp)
132		pass
133		else:
134	<	log.message("No output file defined: only stdout/err will be available")
134	>	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available")
135		pass
136		pass
137		except KeyError:
138	<	log.message("No output file defined: only stdout/err will be available")
138	>	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available")
139		pass
140
141		# script_exe file as additional file in inputSandbox
#	Line 118 | Line 143 | class Cmssw(JobType):
143		self.scriptExe = cfg_params['USER.script_exe']
144		if self.scriptExe != '':
145		if not os.path.isfile(self.scriptExe):
146	<	msg ="WARNING. file "+self.scriptExe+" not found"
146	>	msg ="ERROR. file "+self.scriptExe+" not found"
147		raise CrabException(msg)
148		self.additional_inbox_files.append(string.strip(self.scriptExe))
149		except KeyError:
150		self.scriptExe = ''
151	+
152		#CarlosDaniele
153		if self.datasetPath == None and self.pset == None and self.scriptExe == '' :
154	<	msg ="WARNING. script_exe  not defined"
154	>	msg ="Error. script_exe  not defined"
155		raise CrabException(msg)
156
157		## additional input files
158		try:
159		tmpAddFiles = string.split(cfg_params['USER.additional_input_files'],',')
160		for tmp in tmpAddFiles:
161	+	tmp = string.strip(tmp)
162		dirname = ''
163		if not tmp[0]=="/": dirname = "."
164		files = glob.glob(os.path.join(dirname, tmp))
#	Line 139 | Line 166 | class Cmssw(JobType):
166		if not os.path.exists(file):
167		raise CrabException("Additional input file not found: "+file)
168		pass
169	<	self.additional_inbox_files.append(string.strip(file))
169	>	storedFile = common.work_space.shareDir()+file
170	>	shutil.copyfile(file, storedFile)
171	>	self.additional_inbox_files.append(string.strip(storedFile))
172		pass
173		pass
174		common.logger.debug(5,"Additional input files: "+str(self.additional_inbox_files))
#	Line 197 | Line 226 | class Cmssw(JobType):
226		except KeyError:
227		self.sourceSeedVtx = None
228		common.logger.debug(5,"No vertex seed given")
229	+	try:
230	+	self.firstRun = int(cfg_params['CMSSW.first_run'])
231	+	except KeyError:
232	+	self.firstRun = None
233	+	common.logger.debug(5,"No first run given")
234		if self.pset != None: #CarlosDaniele
235		self.PsetEdit = PsetManipulator.PsetManipulator(self.pset) #Daniele Pset
236
#	Line 220 | Line 254 | class Cmssw(JobType):
254		self.jobSplittingForScript()
255		else:
256		self.jobSplittingNoInput()
257	<	else: self.jobSplittingByBlocks(blockSites)
257	>	else:
258	>	self.jobSplittingByBlocks(blockSites)
259
260		# modify Pset
261		if self.pset != None: #CarlosDaniele
#	Line 232 | Line 267 | class Cmssw(JobType):
267		self.PsetEdit.skipEvent("INPUTSKIPEVENTS")
268		else:  # pythia like job
269		self.PsetEdit.maxEvent(self.eventsPerJob)
270	+	if (self.firstRun):
271	+	self.PsetEdit.pythiaFirstRun("INPUTFIRSTRUN")  #First Run
272		if (self.sourceSeed) :
273		self.PsetEdit.pythiaSeed("INPUT")
274		if (self.sourceSeedVtx) :
275		self.PsetEdit.pythiaSeedVtx("INPUTVTX")
276	+	# add FrameworkJobReport to parameter-set
277	+	self.PsetEdit.addCrabFJR(self.fjrFileName)
278		self.PsetEdit.psetWriter(self.configFilename())
279		except:
280		msg='Error while manipuliating ParameterSet: exiting...'
#	Line 247 | Line 286 | class Cmssw(JobType):
286
287		datasetPath=self.datasetPath
288
250	–	## TODO
251	–	dataTiersList = ""
252	–	dataTiers = dataTiersList.split(',')
253	–
289		## Contact the DBS
290		common.logger.message("Contacting DBS...")
291		try:
292	<	self.pubdata=DataDiscovery.DataDiscovery(datasetPath, dataTiers, cfg_params)
292	>
293	>	if self.use_dbs_2 == 1 :
294	>	self.pubdata=DataDiscovery_DBS2.DataDiscovery_DBS2(datasetPath, cfg_params)
295	>	else :
296	>	self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params)
297		self.pubdata.fetchDBSInfo()
298
299		except DataDiscovery.NotExistingDatasetError, ex :
300		msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
301		raise CrabException(msg)
263	–
302		except DataDiscovery.NoDataTierinProvenanceError, ex :
303		msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
304		raise CrabException(msg)
305		except DataDiscovery.DataDiscoveryError, ex:
306	<	msg = 'ERROR ***: failed Data Discovery in DBS  %s'%ex.getErrorMessage()
306	>	msg = 'ERROR ***: failed Data Discovery in DBS :  %s'%ex.getErrorMessage()
307	>	raise CrabException(msg)
308	>	except DataDiscovery_DBS2.NotExistingDatasetError_DBS2, ex :
309	>	msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
310	>	raise CrabException(msg)
311	>	except DataDiscovery_DBS2.NoDataTierinProvenanceError_DBS2, ex :
312	>	msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
313	>	raise CrabException(msg)
314	>	except DataDiscovery_DBS2.DataDiscoveryError_DBS2, ex:
315	>	msg = 'ERROR ***: failed Data Discovery in DBS :  %s'%ex.getErrorMessage()
316		raise CrabException(msg)
317
318		## get list of all required data in the form of dbs paths  (dbs path = /dataset/datatier/owner)
272	–	## self.DBSPaths=self.pubdata.getDBSPaths()
319		common.logger.message("Required data are :"+self.datasetPath)
320
321		self.filesbyblock=self.pubdata.getFiles()
322		self.eventsbyblock=self.pubdata.getEventsPerBlock()
323		self.eventsbyfile=self.pubdata.getEventsPerFile()
278	–	# print str(self.filesbyblock)
279	–	# print 'self.eventsbyfile',len(self.eventsbyfile)
280	–	# print str(self.eventsbyfile)
324
325		## get max number of events
326		self.maxEvents=self.pubdata.getMaxEvents() ##  self.maxEvents used in Creator.py
#	Line 296 | Line 339 | class Cmssw(JobType):
339		sites = dataloc.getSites()
340		allSites = []
341		listSites = sites.values()
342	<	for list in listSites:
343	<	for oneSite in list:
342	>	for listSite in listSites:
343	>	for oneSite in listSite:
344		allSites.append(oneSite)
345		allSites = self.uniquelist(allSites)
346
#	Line 370 | Line 413 | class Cmssw(JobType):
413		block = blocks[blockCount]
414		blockCount += 1
415
416	<
417	<	numEventsInBlock = self.eventsbyblock[block]
418	<	common.logger.debug(5,'Events in Block File '+str(numEventsInBlock))
416	>	if self.eventsbyblock.has_key(block) :
417	>	numEventsInBlock = self.eventsbyblock[block]
418	>	common.logger.debug(5,'Events in Block File '+str(numEventsInBlock))
419
420	<	files = self.filesbyblock[block]
421	<	numFilesInBlock = len(files)
422	<	if (numFilesInBlock <= 0):
423	<	continue
424	<	fileCount = 0
425	<
426	<	# ---- New block => New job ---- #
427	<	parString = "\\{"
428	<	# counter for number of events in files currently worked on
429	<	filesEventCount = 0
430	<	# flag if next while loop should touch new file
431	<	newFile = 1
432	<	# job event counter
433	<	jobSkipEventCount = 0
420	>	files = self.filesbyblock[block]
421	>	numFilesInBlock = len(files)
422	>	if (numFilesInBlock <= 0):
423	>	continue
424	>	fileCount = 0
425	>
426	>	# ---- New block => New job ---- #
427	>	parString = "\\{"
428	>	# counter for number of events in files currently worked on
429	>	filesEventCount = 0
430	>	# flag if next while loop should touch new file
431	>	newFile = 1
432	>	# job event counter
433	>	jobSkipEventCount = 0
434
435	<	# ---- Iterate over the files in the block until we've met the requested ---- #
436	<	# ---- total # of events or we've gone over all the files in this block  ---- #
437	<	while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) and (jobCount < totalNumberOfJobs) ):
438	<	file = files[fileCount]
439	<	if newFile :
440	<	try:
441	<	numEventsInFile = self.eventsbyfile[file]
442	<	common.logger.debug(6, "File "+str(file)+" has "+str(numEventsInFile)+" events")
443	<	# increase filesEventCount
444	<	filesEventCount += numEventsInFile
445	<	# Add file to current job
446	<	parString += '\\\"' + file + '\\\"\,'
447	<	newFile = 0
448	<	except KeyError:
449	<	common.logger.message("File "+str(file)+" has unknown number of events: skipping")
435	>	# ---- Iterate over the files in the block until we've met the requested ---- #
436	>	# ---- total # of events or we've gone over all the files in this block  ---- #
437	>	while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) and (jobCount < totalNumberOfJobs) ):
438	>	file = files[fileCount]
439	>	if newFile :
440	>	try:
441	>	numEventsInFile = self.eventsbyfile[file]
442	>	common.logger.debug(6, "File "+str(file)+" has "+str(numEventsInFile)+" events")
443	>	# increase filesEventCount
444	>	filesEventCount += numEventsInFile
445	>	# Add file to current job
446	>	parString += '\\\"' + file + '\\\"\,'
447	>	newFile = 0
448	>	except KeyError:
449	>	common.logger.message("File "+str(file)+" has unknown number of events: skipping")
450
451
452	<	# if less events in file remain than eventsPerJobRequested
453	<	if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested ) :
454	<	# if last file in block
455	<	if ( fileCount == numFilesInBlock-1 ) :
456	<	# end job using last file, use remaining events in block
452	>	# if less events in file remain than eventsPerJobRequested
453	>	if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested ) :
454	>	# if last file in block
455	>	if ( fileCount == numFilesInBlock-1 ) :
456	>	# end job using last file, use remaining events in block
457	>	# close job and touch new file
458	>	fullString = parString[:-2]
459	>	fullString += '\\}'
460	>	list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
461	>	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(filesEventCount - jobSkipEventCount)+" events (last file in block).")
462	>	self.jobDestination.append(blockSites[block])
463	>	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
464	>	# reset counter
465	>	jobCount = jobCount + 1
466	>	totalEventCount = totalEventCount + filesEventCount - jobSkipEventCount
467	>	eventsRemaining = eventsRemaining - filesEventCount + jobSkipEventCount
468	>	jobSkipEventCount = 0
469	>	# reset file
470	>	parString = "\\{"
471	>	filesEventCount = 0
472	>	newFile = 1
473	>	fileCount += 1
474	>	else :
475	>	# go to next file
476	>	newFile = 1
477	>	fileCount += 1
478	>	# if events in file equal to eventsPerJobRequested
479	>	elif ( filesEventCount - jobSkipEventCount == eventsPerJobRequested ) :
480		# close job and touch new file
481		fullString = parString[:-2]
482		fullString += '\\}'
483	<	list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
484	<	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(filesEventCount - jobSkipEventCount)+" events (last file in block).")
483	>	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
484	>	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
485		self.jobDestination.append(blockSites[block])
486		common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
487		# reset counter
488		jobCount = jobCount + 1
489	<	totalEventCount = totalEventCount + filesEventCount - jobSkipEventCount
490	<	eventsRemaining = eventsRemaining - filesEventCount + jobSkipEventCount
489	>	totalEventCount = totalEventCount + eventsPerJobRequested
490	>	eventsRemaining = eventsRemaining - eventsPerJobRequested
491		jobSkipEventCount = 0
492		# reset file
493		parString = "\\{"
494		filesEventCount = 0
495		newFile = 1
496		fileCount += 1
497	+
498	+	# if more events in file remain than eventsPerJobRequested
499		else :
500	<	# go to next file
501	<	newFile = 1
502	<	fileCount += 1
503	<	# if events in file equal to eventsPerJobRequested
504	<	elif ( filesEventCount - jobSkipEventCount == eventsPerJobRequested ) :
505	<	# close job and touch new file
506	<	fullString = parString[:-2]
507	<	fullString += '\\}'
508	<	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
509	<	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
510	<	self.jobDestination.append(blockSites[block])
511	<	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
512	<	# reset counter
513	<	jobCount = jobCount + 1
514	<	totalEventCount = totalEventCount + eventsPerJobRequested
515	<	eventsRemaining = eventsRemaining - eventsPerJobRequested
516	<	jobSkipEventCount = 0
517	<	# reset file
518	<	parString = "\\{"
519	<	filesEventCount = 0
452	<	newFile = 1
453	<	fileCount += 1
454	<
455	<	# if more events in file remain than eventsPerJobRequested
456	<	else :
457	<	# close job but don't touch new file
458	<	fullString = parString[:-2]
459	<	fullString += '\\}'
460	<	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
461	<	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
462	<	self.jobDestination.append(blockSites[block])
463	<	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
464	<	# increase counter
465	<	jobCount = jobCount + 1
466	<	totalEventCount = totalEventCount + eventsPerJobRequested
467	<	eventsRemaining = eventsRemaining - eventsPerJobRequested
468	<	# calculate skip events for last file
469	<	# use filesEventCount (contains several files), jobSkipEventCount and eventsPerJobRequest
470	<	jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file])
471	<	# remove all but the last file
472	<	filesEventCount = self.eventsbyfile[file]
473	<	parString = "\\{"
474	<	parString += '\\\"' + file + '\\\"\,'
475	<	pass # END if
476	<	pass # END while (iterate over files in the block)
500	>	# close job but don't touch new file
501	>	fullString = parString[:-2]
502	>	fullString += '\\}'
503	>	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
504	>	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
505	>	self.jobDestination.append(blockSites[block])
506	>	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
507	>	# increase counter
508	>	jobCount = jobCount + 1
509	>	totalEventCount = totalEventCount + eventsPerJobRequested
510	>	eventsRemaining = eventsRemaining - eventsPerJobRequested
511	>	# calculate skip events for last file
512	>	# use filesEventCount (contains several files), jobSkipEventCount and eventsPerJobRequest
513	>	jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file])
514	>	# remove all but the last file
515	>	filesEventCount = self.eventsbyfile[file]
516	>	parString = "\\{"
517	>	parString += '\\\"' + file + '\\\"\,'
518	>	pass # END if
519	>	pass # END while (iterate over files in the block)
520		pass # END while (iterate over blocks in the dataset)
521		self.ncjobs = self.total_number_of_jobs = jobCount
522		if (eventsRemaining > 0 and jobCount < totalNumberOfJobs ):
#	Line 497 | Line 540 | class Cmssw(JobType):
540		raise CrabException(msg)
541
542		if (self.selectEventsPerJob):
543	<	self.total_number_of_jobs = int(self.total_number_of_events/self.eventsPerJob)
543	>	if (self.selectTotalNumberEvents):
544	>	self.total_number_of_jobs = int(self.total_number_of_events/self.eventsPerJob)
545	>	elif(self.selectNumberOfJobs) :
546	>	self.total_number_of_jobs =self.theNumberOfJobs
547	>	self.total_number_of_events =int(self.theNumberOfJobs*self.eventsPerJob)
548	>
549		elif (self.selectNumberOfJobs) :
550		self.total_number_of_jobs = self.theNumberOfJobs
551		self.eventsPerJob = int(self.total_number_of_events/self.total_number_of_jobs)
552	<
552	>
553		common.logger.debug(5,'N jobs  '+str(self.total_number_of_jobs))
554
555		# is there any remainder?
#	Line 519 | Line 567 | class Cmssw(JobType):
567		## Since there is no input, any site is good
568		# self.jobDestination.append(["Any"])
569		self.jobDestination.append([""]) #must be empty to write correctly the xml
570	+	args=''
571	+	if (self.firstRun):
572	+	## pythia first run
573	+	#self.list_of_args.append([(str(self.firstRun)+str(i))])
574	+	args=args+(str(self.firstRun)+str(i))
575	+	else:
576	+	## no first run
577	+	#self.list_of_args.append([str(i)])
578	+	args=args+str(i)
579		if (self.sourceSeed):
580		if (self.sourceSeedVtx):
581		## pythia + vtx random seed
582	<	self.list_of_args.append([
583	<	str(self.sourceSeed)+str(i),
584	<	str(self.sourceSeedVtx)+str(i)
585	<	])
582	>	#self.list_of_args.append([
583	>	#                          str(self.sourceSeed)+str(i),
584	>	#                          str(self.sourceSeedVtx)+str(i)
585	>	#                          ])
586	>	args=args+str(',')+str(self.sourceSeed)+str(i)+str(',')+str(self.sourceSeedVtx)+str(i)
587		else:
588		## only pythia random seed
589	<	self.list_of_args.append([(str(self.sourceSeed)+str(i))])
589	>	#self.list_of_args.append([(str(self.sourceSeed)+str(i))])
590	>	args=args +str(',')+str(self.sourceSeed)+str(i)
591		else:
592		## no random seed
593	<	self.list_of_args.append([str(i)])
594	<	#print self.list_of_args
593	>	if str(args)=='': args=args+(str(self.firstRun)+str(i))
594	>	arguments=args.split(',')
595	>	if len(arguments)==3:self.list_of_args.append([str(arguments[0]),str(arguments[1]),str(arguments[2])])
596	>	elif len(arguments)==2:self.list_of_args.append([str(arguments[0]),str(arguments[1])])
597	>	else :self.list_of_args.append([str(arguments[0])])
598	>
599	>	#   print self.list_of_args
600
601		return
602
#	Line 597 | Line 661 | class Cmssw(JobType):
661		"""
662
663		# if it exist, just return it
664	<	self.tgzNameWithPath = common.work_space.shareDir()+self.tgz_name
664	>	#
665	>	# Marco. Let's start to use relative path for Boss XML files
666	>	#
667	>	self.tgzNameWithPath = common.work_space.pathForTgz()+'share/'+self.tgz_name
668		if os.path.exists(self.tgzNameWithPath):
669		return self.tgzNameWithPath
670
#	Line 611 | Line 678 | class Cmssw(JobType):
678		# First of all declare the user Scram area
679		swArea = self.scram.getSWArea_()
680		#print "swArea = ", swArea
681	<	swVersion = self.scram.getSWVersion()
682	<	#print "swVersion = ", swVersion
681	>	# swVersion = self.scram.getSWVersion()
682	>	# print "swVersion = ", swVersion
683		swReleaseTop = self.scram.getReleaseTop_()
684		#print "swReleaseTop = ", swReleaseTop
685
#	Line 620 | Line 687 | class Cmssw(JobType):
687		if swReleaseTop == '' or swArea == swReleaseTop:
688		return
689
690	<	filesToBeTarred = []
691	<	## First find the executable
692	<	if (self.executable != ''):
693	<	exeWithPath = self.scram.findFile_(executable)
694	<	#           print exeWithPath
695	<	if ( not exeWithPath ):
696	<	raise CrabException('User executable '+executable+' not found')
697	<
698	<	## then check if it's private or not
699	<	if exeWithPath.find(swReleaseTop) == -1:
700	<	# the exe is private, so we must ship
701	<	common.logger.debug(5,"Exe "+exeWithPath+" to be tarred")
702	<	path = swArea+'/'
703	<	exe = string.replace(exeWithPath, path,'')
704	<	filesToBeTarred.append(exe)
705	<	pass
706	<	else:
707	<	# the exe is from release, we'll find it on WN
708	<	pass
709	<
710	<	## Now get the libraries: only those in local working area
711	<	libDir = 'lib'
712	<	lib = swArea+'/' +libDir
713	<	common.logger.debug(5,"lib "+lib+" to be tarred")
714	<	if os.path.exists(lib):
715	<	filesToBeTarred.append(libDir)
716	<
717	<	## Now check if module dir is present
718	<	moduleDir = 'module'
719	<	if os.path.isdir(swArea+'/'+moduleDir):
720	<	filesToBeTarred.append(moduleDir)
721	<
722	<	## Now check if the Data dir is present
723	<	dataDir = 'src/Data/'
724	<	if os.path.isdir(swArea+'/'+dataDir):
725	<	filesToBeTarred.append(dataDir)
726	<
727	<	## Create the tar-ball
728	<	if len(filesToBeTarred)>0:
729	<	cwd = os.getcwd()
730	<	os.chdir(swArea)
731	<	tarcmd = 'tar zcvf ' + self.tgzNameWithPath + ' '
732	<	for line in filesToBeTarred:
733	<	tarcmd = tarcmd + line + ' '
734	<	cout = runCommand(tarcmd)
735	<	if not cout:
736	<	raise CrabException('Could not create tar-ball')
737	<	os.chdir(cwd)
738	<	else:
739	<	common.logger.debug(5,"No files to be to be tarred")
690	>	import tarfile
691	>	try: # create tar ball
692	>	tar = tarfile.open(self.tgzNameWithPath, "w:gz")
693	>	## First find the executable
694	>	if (executable != ''):
695	>	exeWithPath = self.scram.findFile_(executable)
696	>	if ( not exeWithPath ):
697	>	raise CrabException('User executable '+executable+' not found')
698	>
699	>	## then check if it's private or not
700	>	if exeWithPath.find(swReleaseTop) == -1:
701	>	# the exe is private, so we must ship
702	>	common.logger.debug(5,"Exe "+exeWithPath+" to be tarred")
703	>	path = swArea+'/'
704	>	# distinguish case when script is in user project area or given by full path somewhere else
705	>	if exeWithPath.find(path) >= 0 :
706	>	exe = string.replace(exeWithPath, path,'')
707	>	tar.add(path+exe,os.path.basename(executable))
708	>	else :
709	>	tar.add(exeWithPath,os.path.basename(executable))
710	>	pass
711	>	else:
712	>	# the exe is from release, we'll find it on WN
713	>	pass
714	>
715	>	## Now get the libraries: only those in local working area
716	>	libDir = 'lib'
717	>	lib = swArea+'/' +libDir
718	>	common.logger.debug(5,"lib "+lib+" to be tarred")
719	>	if os.path.exists(lib):
720	>	tar.add(lib,libDir)
721	>
722	>	## Now check if module dir is present
723	>	moduleDir = 'module'
724	>	module = swArea + '/' + moduleDir
725	>	if os.path.isdir(module):
726	>	tar.add(module,moduleDir)
727	>
728	>	## Now check if any data dir(s) is present
729	>	swAreaLen=len(swArea)
730	>	for root, dirs, files in os.walk(swArea):
731	>	if "data" in dirs:
732	>	common.logger.debug(5,"data "+root+"/data"+" to be tarred")
733	>	tar.add(root+"/data",root[swAreaLen:]+"/data")
734	>
735	>	## Add ProdAgent dir to tar
736	>	paDir = 'ProdAgentApi'
737	>	pa = os.environ['CRABDIR'] + '/' + 'ProdAgentApi'
738	>	if os.path.isdir(pa):
739	>	tar.add(pa,paDir)
740	>
741	>	common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
742	>	tar.close()
743	>	except :
744	>	raise CrabException('Could not create tar-ball')
745	>
746	>	## check for tarball size
747	>	tarballinfo = os.stat(self.tgzNameWithPath)
748	>	if ( tarballinfo.st_size > self.MaxTarBallSize*1024*1024 ) :
749	>	raise CrabException('Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' + str(self.MaxTarBallSize) + ' MB input sandbox limit and not supported by the used GRID submission system. Please make sure that no unnecessary files are in all data directories in your local CMSSW project area as they are automatically packed into the input sandbox.')
750	>
751	>	## create tar-ball with ML stuff
752	>	self.MLtgzfile =  common.work_space.pathForTgz()+'share/MLfiles.tgz'
753	>	try:
754	>	tar = tarfile.open(self.MLtgzfile, "w:gz")
755	>	path=os.environ['CRABDIR'] + '/python/'
756	>	for file in ['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py', 'parseCrabFjr.py']:
757	>	tar.add(path+file,file)
758	>	common.logger.debug(5,"Files added to "+self.MLtgzfile+" : "+str(tar.getnames()))
759	>	tar.close()
760	>	except :
761	>	raise CrabException('Could not create ML files tar-ball')
762
763		return
764
#	Line 736 | Line 825 | class Cmssw(JobType):
825		txt += '   exit 1 \n'
826		txt += 'fi \n'
827		txt += 'echo "CMSSW_VERSION =  '+self.version+'"\n'
828	+	txt += 'export SCRAM_ARCH='+self.executable_arch+'\n'
829		txt += 'cd '+self.version+'\n'
830		### needed grep for bug in scramv1 ###
831	+	txt += scram+' runtime -sh\n'
832		txt += 'eval `'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME`\n'
833	+	txt += 'echo $PATH\n'
834
835		# Handle the arguments:
836		txt += "\n"
#	Line 791 | Line 883 | class Cmssw(JobType):
883		txt += 'sed "s#INPUTSKIPEVENTS#$SkipEvents#" pset_tmp_2.cfg > pset.cfg\n'
884		else:  # pythia like job
885		if (self.sourceSeed):
886	+	txt += 'FirstRun=${args[1]}\n'
887	+	txt += 'echo "FirstRun: <$FirstRun>"\n'
888	+	txt += 'sed "s#\<INPUTFIRSTRUN\>#$FirstRun#" $RUNTIME_AREA/'+pset+' > tmp_1.cfg\n'
889	+	else:
890	+	txt += '# Copy untouched pset\n'
891	+	txt += 'cp $RUNTIME_AREA/'+pset+' tmp_1.cfg\n'
892	+	if (self.sourceSeed):
893		#                    txt += 'Seed=$2\n'
894	<	txt += 'Seed=${args[1]}\n'
894	>	txt += 'Seed=${args[2]}\n'
895		txt += 'echo "Seed: <$Seed>"\n'
896	<	txt += 'sed "s#\<INPUT\>#$Seed#" $RUNTIME_AREA/'+pset+' > tmp.cfg\n'
896	>	txt += 'sed "s#\<INPUT\>#$Seed#" tmp_1.cfg > tmp_2.cfg\n'
897		if (self.sourceSeedVtx):
898		#                        txt += 'VtxSeed=$3\n'
899	<	txt += 'VtxSeed=${args[2]}\n'
899	>	txt += 'VtxSeed=${args[3]}\n'
900		txt += 'echo "VtxSeed: <$VtxSeed>"\n'
901	<	txt += 'sed "s#INPUTVTX#$VtxSeed#" tmp.cfg > pset.cfg\n'
901	>	txt += 'sed "s#INPUTVTX#$VtxSeed#" tmp_2.cfg > pset.cfg\n'
902		else:
903	<	txt += 'mv tmp.cfg pset.cfg\n'
903	>	txt += 'mv tmp_2.cfg pset.cfg\n'
904		else:
905	<	txt += '# Copy untouched pset\n'
906	<	txt += 'cp $RUNTIME_AREA/'+pset+' pset.cfg\n'
905	>	txt += 'mv tmp_1.cfg pset.cfg\n'
906	>	# txt += '# Copy untouched pset\n'
907	>	# txt += 'cp $RUNTIME_AREA/'+pset+' pset.cfg\n'
908
909
910		if len(self.additional_inbox_files) > 0:
#	Line 829 | Line 929 | class Cmssw(JobType):
929		# txt += 'echo "****** end pset1.cfg ********"\n'
930		return txt
931
932	<	def wsBuildExe(self, nj):
932	>	def wsBuildExe(self, nj=0):
933		"""
934		Put in the script the commands to build an executable
935		or a library.
#	Line 864 | Line 964 | class Cmssw(JobType):
964		txt += 'else \n'
965		txt += '   echo "Successful untar" \n'
966		txt += 'fi \n'
967	+	txt += '\n'
968	+	txt += 'echo "Include ProdAgentApi in PYTHONPATH"\n'
969	+	txt += 'if [ -z "$PYTHONPATH" ]; then\n'
970	+	txt += '   export PYTHONPATH=ProdAgentApi\n'
971	+	txt += 'else\n'
972	+	txt += '   export PYTHONPATH=ProdAgentApi:${PYTHONPATH}\n'
973	+	txt += 'fi\n'
974	+	txt += '\n'
975	+
976		pass
977
978		return txt
#	Line 875 | Line 984 | class Cmssw(JobType):
984		"""
985
986		def executableName(self):
987	<	if self.pset == None: #CarlosDaniele
987	>	if self.scriptExe: #CarlosDaniele
988		return "sh "
989		else:
990		return self.executable
991
992		def executableArgs(self):
993	<	if self.pset == None:#CarlosDaniele
993	>	if self.scriptExe:#CarlosDaniele
994		return   self.scriptExe + " $NJob"
995		else:
996		return " -p pset.cfg"
#	Line 891 | Line 1000 | class Cmssw(JobType):
1000		Returns a list of filenames to be put in JDL input sandbox.
1001		"""
1002		inp_box = []
1003	<	# dict added to delete duplicate from input sandbox file list
1004	<	seen = {}
1003	>	# # dict added to delete duplicate from input sandbox file list
1004	>	# seen = {}
1005		## code
1006		if os.path.isfile(self.tgzNameWithPath):
1007		inp_box.append(self.tgzNameWithPath)
1008	+	if os.path.isfile(self.MLtgzfile):
1009	+	inp_box.append(self.MLtgzfile)
1010		## config
1011	<	if not self.pset is None: #CarlosDaniele
1012	<	inp_box.append(common.job_list[nj].configFilename())
1011	>	if not self.pset is None:
1012	>	inp_box.append(common.work_space.pathForTgz() + 'job/' + self.configFilename())
1013		## additional input files
1014	<	#for file in self.additional_inbox_files:
1015	<	#    inp_box.append(common.work_space.cwdDir()+file)
1014	>	for file in self.additional_inbox_files:
1015	>	inp_box.append(file)
1016		return inp_box
1017
1018		def outputSandbox(self, nj):
#	Line 910 | Line 1021 | class Cmssw(JobType):
1021		"""
1022		out_box = []
1023
913	–	stdout=common.job_list[nj].stdout()
914	–	stderr=common.job_list[nj].stderr()
915	–
1024		## User Declared output files
1025	<	for out in self.output_file:
1025	>	for out in (self.output_file+self.output_file_sandbox):
1026		n_out = nj + 1
1027		out_box.append(self.numberFile_(out,str(n_out)))
1028		return out_box
921	–	return []
1029
1030		def prepareSteeringCards(self):
1031		"""
#	Line 934 | Line 1041 | class Cmssw(JobType):
1041		txt = '\n'
1042		txt += '# directory content\n'
1043		txt += 'ls \n'
1044	<	file_list = ''
1045	<	for fileWithSuffix in self.output_file:
1044	>
1045	>	for fileWithSuffix in (self.output_file+self.output_file_sandbox):
1046		output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
940	–	file_list=file_list+output_file_num+' '
1047		txt += '\n'
1048		txt += '# check output file\n'
1049		txt += 'ls '+fileWithSuffix+'\n'
1050		txt += 'ls_result=$?\n'
945	–	#txt += 'exe_result=$?\n'
1051		txt += 'if [ $ls_result -ne 0 ] ; then\n'
1052		txt += '   echo "ERROR: Problem with output file"\n'
948	–	#txt += '   echo "JOB_EXIT_STATUS = $exe_result"\n'
949	–	#txt += '   echo "JobExitCode=60302" | tee -a $RUNTIME_AREA/$repo\n'
950	–	#txt += '   dumpStatus $RUNTIME_AREA/$repo\n'
951	–	### OLI_DANIELE
1053		if common.scheduler.boss_scheduler_name == 'condor_g':
1054		txt += '    if [ $middleware == OSG ]; then \n'
1055		txt += '        echo "prepare dummy output file"\n'
#	Line 959 | Line 1060 | class Cmssw(JobType):
1060		txt += 'fi\n'
1061
1062		txt += 'cd $RUNTIME_AREA\n'
962	–	file_list=file_list[:-1]
963	–	txt += 'file_list="'+file_list+'"\n'
1063		txt += 'cd $RUNTIME_AREA\n'
1064		### OLI_DANIELE
1065		txt += 'if [ $middleware == OSG ]; then\n'
#	Line 978 | Line 1077 | class Cmssw(JobType):
1077		txt += '    fi\n'
1078		txt += 'fi\n'
1079		txt += '\n'
1080	+
1081	+	file_list = ''
1082	+	## Add to filelist only files to be possibly copied to SE
1083	+	for fileWithSuffix in self.output_file:
1084	+	output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1085	+	file_list=file_list+output_file_num+' '
1086	+	file_list=file_list[:-1]
1087	+	txt += 'file_list="'+file_list+'"\n'
1088	+
1089		return txt
1090
1091		def numberFile_(self, file, txt):
#	Line 998 | Line 1106 | class Cmssw(JobType):
1106
1107		return result
1108
1109	<	def getRequirements(self):
1109	>	def getRequirements(self, nj=[]):
1110		"""
1111		return job requirements to add to jdl files
1112		"""
#	Line 1105 | Line 1213 | class Cmssw(JobType):
1213		txt += '       fi\n'
1214		txt += '   fi\n'
1215		txt += '   \n'
1108	–	txt += '   string=`cat /etc/redhat-release`\n'
1109	–	txt += '   echo $string\n'
1110	–	txt += '   if [[ $string = *alhalla* ]]; then\n'
1111	–	txt += '       echo "SCRAM_ARCH= $SCRAM_ARCH"\n'
1112	–	txt += '   elif [[ $string = *Enterprise* ]] || [[ $string = *cientific* ]]; then\n'
1113	–	txt += '       export SCRAM_ARCH=slc3_ia32_gcc323\n'
1114	–	txt += '       echo "SCRAM_ARCH= $SCRAM_ARCH"\n'
1115	–	txt += '   else\n'
1116	–	txt += '       echo "SET_CMS_ENV 10033 ==> ERROR OS unknown, LCG environment not initialized"\n'
1117	–	txt += '       echo "JOB_EXIT_STATUS = 10033"\n'
1118	–	txt += '       echo "JobExitCode=10033" | tee -a $RUNTIME_AREA/$repo\n'
1119	–	txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
1120	–	txt += '       rm -f $RUNTIME_AREA/$repo \n'
1121	–	txt += '       echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1122	–	txt += '       echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1123	–	txt += '       exit 1\n'
1124	–	txt += '   fi\n'
1216		txt += '   echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1217		txt += '   echo "### END SETUP CMS LCG ENVIRONMENT ###"\n'
1218		return txt

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines