ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/DataDiscovery.py

Comparing COMP/CRAB/python/DataDiscovery.py (file contents):
Revision 1.14 by slacapra, Wed Jan 17 18:17:58 2007 UTC vs.
Revision 1.20 by afanfani, Fri Jan 11 22:11:55 2008 UTC

#	Line 1 | Line 1
1		#!/usr/bin/env python
2	<	from DBSInfo import *
2	>	import exceptions
3	>	import DBSAPI.dbsApi
4	>	from DBSAPI.dbsApiException import *
5	>	import common
6	>	from crab_util import *
7
8
9	+	# #######################################
10	+	class DBSError(exceptions.Exception):
11	+	def __init__(self, errorName, errorMessage):
12	+	args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
13	+	exceptions.Exception.__init__(self, args)
14	+	pass
15	+
16	+	def getErrorMessage(self):
17	+	""" Return error message """
18	+	return "%s" % (self.args)
19	+
20	+	# #######################################
21	+	class DBSInvalidDataTierError(exceptions.Exception):
22	+	def __init__(self, errorName, errorMessage):
23	+	args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
24	+	exceptions.Exception.__init__(self, args)
25	+	pass
26	+
27	+	def getErrorMessage(self):
28	+	""" Return error message """
29	+	return "%s" % (self.args)
30	+
31	+	# #######################################
32	+	class DBSInfoError:
33	+	def __init__(self, url):
34	+	print '\nERROR accessing DBS url : '+url+'\n'
35	+	pass
36	+
37		# ####################################
38		class DataDiscoveryError(exceptions.Exception):
39		def __init__(self, errorMessage):
8	–	exceptions.Exception.__init__(self, self.args)
40		self.args=errorMessage
41	+	exceptions.Exception.__init__(self, self.args)
42		pass
43
44		def getErrorMessage(self):
#	Line 16 | Line 48 | class DataDiscoveryError(exceptions.Exce
48		# ####################################
49		class NotExistingDatasetError(exceptions.Exception):
50		def __init__(self, errorMessage):
19	–	exceptions.Exception.__init__(self, self.args)
51		self.args=errorMessage
52	+	exceptions.Exception.__init__(self, self.args)
53		pass
54
55		def getErrorMessage(self):
#	Line 27 | Line 59 | class NotExistingDatasetError(exceptions
59		# ####################################
60		class NoDataTierinProvenanceError(exceptions.Exception):
61		def __init__(self, errorMessage):
30	–	exceptions.Exception.__init__(self, self.args)
62		self.args=errorMessage
63	+	exceptions.Exception.__init__(self, self.args)
64		pass
65
66		def getErrorMessage(self):
#	Line 38 | Line 70 | class NoDataTierinProvenanceError(except
70		# ####################################
71		# class to find and extact info from published data
72		class DataDiscovery:
73	<	def __init__(self, datasetPath, dataTiers, cfg_params):
73	>	def __init__(self, datasetPath, cfg_params):
74
75	<	#       Attributes
75	>	#       Attributes
76		self.datasetPath = datasetPath
45	–	self.dataTiers = dataTiers
77		self.cfg_params = cfg_params
78
79		self.eventsPerBlock = {}  # DBS output: map fileblocks-events for collection
80		self.eventsPerFile = {}   # DBS output: map files-events
81	<	self.blocksinfo = {}  # DBS output: map fileblocks-files
82	<	#DBS output: max events computed by method getMaxEvents
81	>	self.blocksinfo = {}      # DBS output: map fileblocks-files
82	>	self.maxEvents = 0        # DBS output: max events
83
84		# ####################################
85		def fetchDBSInfo(self):
#	Line 57 | Line 88 | class DataDiscovery:
88		"""
89
90		## get DBS URL
91	<	try:
91	>	dbs_url="http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
92	>	if (self.cfg_params.has_key('CMSSW.dbs_url')):
93		dbs_url=self.cfg_params['CMSSW.dbs_url']
62	–	except KeyError:
63	–	dbs_url="http://cmsdoc.cern.ch/cms/test/aprom/DBS/CGIServer/prodquery"
94
95	<	## get info about the requested dataset
96	<	try:
97	<	dbs_instance=self.cfg_params['CMSSW.dbs_instance']
98	<	except KeyError:
99	<	dbs_instance="MCGlobal/Writer"
100	<
101	<	dbs = DBSInfo(dbs_url, dbs_instance)
95	>	common.logger.debug(3,"Accessing DBS at: "+dbs_url)
96	>
97	>	## check if runs are selected
98	>	runselection = []
99	>	if (self.cfg_params.has_key('CMSSW.runselection')):
100	>	runselection = parseRange2(self.cfg_params['CMSSW.runselection'])
101	>
102	>	common.logger.debug(6,"runselection is: %s"%runselection)
103	>	## service API
104	>	args = {}
105	>	args['url']     = dbs_url
106	>	args['level']   = 'CRITICAL'
107	>
108	>	api = DBSAPI.dbsApi.DbsApi(args)
109		try:
110	<	self.datasets = dbs.getMatchingDatasets(self.datasetPath)
111	<	except dbsCgiApi.DbsCgiExecutionError, msg:
110	>	if len(runselection) <= 0 :
111	>	files = api.listDatasetFiles(self.datasetPath)
112	>	else :
113	>	files=[]
114	>	for arun in runselection:
115	>	try:
116	>	filesinrun = api.listFiles(path=self.datasetPath, details=True,runNumber=arun)
117	>	files.extend(filesinrun)
118	>	except:
119	>	msg="WARNING: problem extracting info from DBS for run %s "%arun
120	>	common.logger.message(msg)
121	>	pass
122	>
123	>	except DbsBadRequest, msg:
124		raise DataDiscoveryError(msg)
125		except DBSError, msg:
126		raise DataDiscoveryError(msg)
127
128	<	if len(self.datasets) == 0:
129	<	raise DataDiscoveryError("DatasetPath=%s unknown to DBS" %self.datasetPath)
130	<	if len(self.datasets) > 1:
131	<	raise DataDiscoveryError("DatasetPath=%s is ambiguous" %self.datasetPath)
128	>	# parse files and fill arrays
129	>	for file in files :
130	>	filename = file['LogicalFileName']
131	>	if filename.find('.dat') < 0 :
132	>	fileblock = file['Block']['Name']
133	>	events    = file['NumberOfEvents']
134	>	# number of events per block
135	>	if fileblock in self.eventsPerBlock.keys() :
136	>	self.eventsPerBlock[fileblock] += events
137	>	else :
138	>	self.eventsPerBlock[fileblock] = events
139	>	# number of events per file
140	>	self.eventsPerFile[filename] = events
141	>
142	>	# number of events per block
143	>	if fileblock in self.blocksinfo.keys() :
144	>	self.blocksinfo[fileblock].append(filename)
145	>	else :
146	>	self.blocksinfo[fileblock] = [filename]
147
148	<	try:
149	<	self.dbsdataset = self.datasets[0].get('datasetPathName')
148	>	# total number of events
149	>	self.maxEvents += events
150	>
151	>	for block in self.eventsPerBlock.keys() :
152	>	common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(self.eventsPerBlock[block],block))
153
87	–	self.eventsPerBlock = dbs.getEventsPerBlock(self.dbsdataset)
88	–	self.blocksinfo = dbs.getDatasetFileBlocks(self.dbsdataset)
89	–	self.eventsPerFile = dbs.getEventsPerFile(self.dbsdataset)
90	–	except DBSError, ex:
91	–	raise DataDiscoveryError(ex.getErrorMessage())
92	–
154		if len(self.eventsPerBlock) <= 0:
155	<	raise NotExistingDatasetError (("\nNo data for %s in DBS\nPlease check"
155	>	raise NotExistingDatasetError(("\nNo data for %s in DBS\nPlease check"
156		+ " dataset path variables in crab.cfg")
157	<	% self.dbsdataset)
157	>	% self.datasetPath)
158
159
160		# #################################################
#	Line 101 | Line 162 | class DataDiscovery:
162		"""
163		max events
164		"""
165	<	## loop over the event collections
105	<	nevts=0
106	<	for evc_evts in self.eventsPerBlock.values():
107	<	nevts=nevts+evc_evts
108	<
109	<	return nevts
165	>	return self.maxEvents
166
167		# #################################################
168		def getEventsPerBlock(self):

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines