ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/Splitter.py

Comparing COMP/CRAB/python/Splitter.py (file contents):
Revision 1.22 by ewv, Wed Jun 17 20:58:07 2009 UTC vs.
Revision 1.24 by spiga, Tue Jul 21 16:18:09 2009 UTC

#	Line 9 | Line 9 | class JobSplitter:
9		self.args=args
10		#self.maxEvents
11		# init BlackWhiteListParser
12	<	seWhiteList = cfg_params.get('GRID.se_white_list',[])
12	>	self.seWhiteList = cfg_params.get('GRID.se_white_list',[])
13		seBlackList = cfg_params.get('GRID.se_black_list',[])
14	<	self.blackWhiteListParser = SEBlackWhiteListParser(seWhiteList, seBlackList, common.logger())
14	>	self.blackWhiteListParser = SEBlackWhiteListParser(self.seWhiteList, seBlackList, common.logger())
15
16
17		def checkUserSettings(self):
#	Line 43 | Line 43 | class JobSplitter:
43		self.selectTotalNumberEvents = 0
44
45
46	+	def ComputeSubBlockSites( self, blockSites ):
47	+	"""
48	+	"""
49	+	sub_blockSites = {}
50	+	for k,v in blockSites.iteritems():
51	+	sites=self.blackWhiteListParser.checkWhiteList(v)
52	+	if sites : sub_blockSites[k]=v
53	+	if len(sub_blockSites) < 1:
54	+	msg = 'WARNING: the sites %s is not hosting any part of data.'%self.seWhiteList
55	+	raise CrabException(msg)
56	+	return sub_blockSites
57	+
58		########################################################################
59		def jobSplittingByEvent( self ):
60		"""
#	Line 77 | Line 89 | class JobSplitter:
89		self.useParent = int(self.cfg_params.get('CMSSW.use_parent',0))
90		noBboundary = int(self.cfg_params.get('CMSSW.no_block_boundary',0))
91
92	+	if noBboundary == 1:
93	+	if self.total_number_of_events== -1:
94	+	msg = 'You are selecting no_block_boundary=1 which does not allow to set total_number_of_events=-1\n'
95	+	msg +='\tYou shoud get the number of event from DBS web interface and use it for your configuration.'
96	+	raise CrabException(msg)
97	+	if len(self.seWhiteList.split(',')) != 1:
98	+	msg = 'You are selecting no_block_boundary=1 which requires to choose one and only one site.\n'
99	+	msg += "\tPlease set se_white_list with the site's storage element name."
100	+	raise  CrabException(msg)
101	+	blockSites = self.ComputeSubBlockSites(blockSites)
102	+
103		# ---- Handle the possible job splitting configurations ---- #
104		if (self.selectTotalNumberEvents):
105		totalEventsRequested = self.total_number_of_events
#	Line 435 | Line 458 | class JobSplitter:
458		return res
459
460		########################################################################
461	<	def jobSplittingNoInput(self):
461	>	def prepareSplittingNoInput(self):
462		"""
440	–	Perform job splitting based on number of event per job
463		"""
442	–	common.logger.debug('Splitting per events')
443	–	self.checkUserSettings()
444	–	jobDestination=[]
445	–	if ( (self.selectTotalNumberEvents + self.selectEventsPerJob + self.selectNumberOfJobs) != 2 ):
446	–	msg = 'Must define exactly two of total_number_of_events, events_per_job, or number_of_jobs.'
447	–	raise CrabException(msg)
448	–
449	–	managedGenerators =self.args['managedGenerators']
450	–	generator = self.args['generator']
451	–	firstRun = self.cfg_params.get('CMSSW.first_run',None)
452	–
464		if (self.selectEventsPerJob):
465		common.logger.info('Required '+str(self.eventsPerJob)+' events per job ')
466		if (self.selectNumberOfJobs):
#	Line 472 | Line 483 | class JobSplitter:
483		self.total_number_of_jobs = self.theNumberOfJobs
484		self.eventsPerJob = int(self.total_number_of_events/self.total_number_of_jobs)
485
486	+
487	+	def jobSplittingNoInput(self):
488	+	"""
489	+	Perform job splitting based on number of event per job
490	+	"""
491	+	common.logger.debug('Splitting per events')
492	+	self.checkUserSettings()
493	+	jobDestination=[]
494	+	if ( (self.selectTotalNumberEvents + self.selectEventsPerJob + self.selectNumberOfJobs) != 2 ):
495	+	msg = 'Must define exactly two of total_number_of_events, events_per_job, or number_of_jobs.'
496	+	raise CrabException(msg)
497	+
498	+	managedGenerators =self.args['managedGenerators']
499	+	generator = self.args['generator']
500	+	firstRun = self.cfg_params.get('CMSSW.first_run',None)
501	+
502	+	self.prepareSplittingNoInput()
503	+
504		common.logger.debug('N jobs  '+str(self.total_number_of_jobs))
505
506		# is there any remainder?
#	Line 531 | Line 560 | class JobSplitter:
560		common.logger.debug('Splitting per job')
561		common.logger.info('Required '+str(self.theNumberOfJobs)+' jobs in total ')
562
563	<	self.total_number_of_jobs = self.theNumberOfJobs
563	>	#        self.total_number_of_jobs = self.theNumberOfJobs
564	>
565	>	self.prepareSplittingNoInput()
566
567		common.logger.debug('N jobs  '+str(self.total_number_of_jobs))
568
569		common.logger.info(str(self.total_number_of_jobs)+' jobs can be created')
570
571		# argument is seed number.$i
572	<	#self.list_of_args = []
572	>	self.list_of_args = []
573		for i in range(self.total_number_of_jobs):
574	+	args=[]
575		jobDestination.append([""])
576	<	#   self.list_of_args.append([str(i)])
576	>	if self.eventsPerJob != 0 :
577	>	args.append(str(self.eventsPerJob))
578	>	self.list_of_args.append(args)
579
580		# prepare dict output
581		dictOut = {}
582	<	dictOut['args'] = [] # self.list_of_args
582	>	dictOut['params'] = ['MaxEvents']
583	>	dictOut['args'] =  self.list_of_args
584		dictOut['jobDestination'] = jobDestination
585		dictOut['njobs']=self.total_number_of_jobs
586		return dictOut

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines