ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/DataDiscovery.py

Comparing COMP/CRAB/python/DataDiscovery.py (file contents):
Revision 1.13 by slacapra, Mon Dec 18 12:28:06 2006 UTC vs.
Revision 1.21 by spiga, Thu Jun 5 07:13:31 2008 UTC

#	Line 1 | Line 1
1		#!/usr/bin/env python
2	<	import sys, os, string, re
3	<	from DBSInfo import *
2	>	import exceptions
3	>	import DBSAPI.dbsApi
4	>	from DBSAPI.dbsApiException import *
5	>	import common
6	>	from crab_util import *
7
8
9	+	# #######################################
10	+	class DBSError(exceptions.Exception):
11	+	def __init__(self, errorName, errorMessage):
12	+	args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
13	+	exceptions.Exception.__init__(self, args)
14	+	pass
15	+
16	+	def getErrorMessage(self):
17	+	""" Return error message """
18	+	return "%s" % (self.args)
19	+
20	+	# #######################################
21	+	class DBSInvalidDataTierError(exceptions.Exception):
22	+	def __init__(self, errorName, errorMessage):
23	+	args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
24	+	exceptions.Exception.__init__(self, args)
25	+	pass
26	+
27	+	def getErrorMessage(self):
28	+	""" Return error message """
29	+	return "%s" % (self.args)
30	+
31	+	# #######################################
32	+	class DBSInfoError:
33	+	def __init__(self, url):
34	+	print '\nERROR accessing DBS url : '+url+'\n'
35	+	pass
36	+
37		# ####################################
38		class DataDiscoveryError(exceptions.Exception):
39		def __init__(self, errorMessage):
40	<	args=errorMessage
41	<	exceptions.Exception.__init__(self, args)
40	>	self.args=errorMessage
41	>	exceptions.Exception.__init__(self, self.args)
42		pass
43
44		def getErrorMessage(self):
#	Line 17 | Line 48 | class DataDiscoveryError(exceptions.Exce
48		# ####################################
49		class NotExistingDatasetError(exceptions.Exception):
50		def __init__(self, errorMessage):
51	<	args=errorMessage
52	<	exceptions.Exception.__init__(self, args)
51	>	self.args=errorMessage
52	>	exceptions.Exception.__init__(self, self.args)
53		pass
54
55		def getErrorMessage(self):
#	Line 28 | Line 59 | class NotExistingDatasetError(exceptions
59		# ####################################
60		class NoDataTierinProvenanceError(exceptions.Exception):
61		def __init__(self, errorMessage):
62	<	args=errorMessage
63	<	exceptions.Exception.__init__(self, args)
62	>	self.args=errorMessage
63	>	exceptions.Exception.__init__(self, self.args)
64		pass
65
66		def getErrorMessage(self):
#	Line 39 | Line 70 | class NoDataTierinProvenanceError(except
70		# ####################################
71		# class to find and extact info from published data
72		class DataDiscovery:
73	<	def __init__(self, datasetPath, dataTiers, cfg_params):
73	>	def __init__(self, datasetPath, cfg_params):
74
75	<	#       Attributes
75	>	#       Attributes
76		self.datasetPath = datasetPath
46	–	self.dataTiers = dataTiers
77		self.cfg_params = cfg_params
78
79		self.eventsPerBlock = {}  # DBS output: map fileblocks-events for collection
80		self.eventsPerFile = {}   # DBS output: map files-events
81	<	self.blocksinfo = {}  # DBS output: map fileblocks-files
82	<	#DBS output: max events computed by method getMaxEvents
81	>	self.blocksinfo = {}      # DBS output: map fileblocks-files
82	>	self.maxEvents = 0        # DBS output: max events
83	>	self.parent = {}       # DBS output: max events
84
85		# ####################################
86		def fetchDBSInfo(self):
#	Line 58 | Line 89 | class DataDiscovery:
89		"""
90
91		## get DBS URL
92	<	try:
92	>	dbs_url="http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
93	>	if (self.cfg_params.has_key('CMSSW.dbs_url')):
94		dbs_url=self.cfg_params['CMSSW.dbs_url']
63	–	except KeyError:
64	–	dbs_url="http://cmsdoc.cern.ch/cms/test/aprom/DBS/CGIServer/prodquery"
95
96	<	## get info about the requested dataset
97	<	try:
98	<	dbs_instance=self.cfg_params['CMSSW.dbs_instance']
99	<	except KeyError:
100	<	dbs_instance="MCGlobal/Writer"
101	<
102	<	dbs = DBSInfo(dbs_url, dbs_instance)
96	>	common.logger.debug(3,"Accessing DBS at: "+dbs_url)
97	>
98	>	## check if runs are selected
99	>	runselection = []
100	>	if (self.cfg_params.has_key('CMSSW.runselection')):
101	>	runselection = parseRange2(self.cfg_params['CMSSW.runselection'])
102	>
103	>	common.logger.debug(6,"runselection is: %s"%runselection)
104	>	## service API
105	>	args = {}
106	>	args['url']     = dbs_url
107	>	args['level']   = 'CRITICAL'
108	>
109	>	## check if has been requested to use the parent info
110	>	if (self.cfg_params.has_key('CMSSW.runselection')):
111	>	runselection = parseRange2(self.cfg_params['CMSSW.runselection'])
112	>
113	>	useParent = self.cfg_params.get('CMSSW.use_parent',False)
114	>
115	>	allowedRetriveValue = [
116	>	'retrive_child',
117	>	'retrive_block',
118	>	'retrive_lumi',
119	>	'retrive_run'
120	>	]
121	>	if useParent:  allowedRetriveValue.append('retrive_parent')
122	>	common.logger.debug(5,"Set of input parameters used for DBS query : \n"+str(allowedRetriveValue))
123	>	common.logger.write("Set of input parameters used for DBS query : \n"+str(allowedRetriveValue))
124	>	api = DBSAPI.dbsApi.DbsApi(args)
125		try:
126	<	self.datasets = dbs.getMatchingDatasets(self.datasetPath)
127	<	except dbsCgiApi.DbsCgiExecutionError, msg:
126	>	if len(runselection) <= 0 :
127	>	files = api.listFiles(path=self.datasetPath,retriveList=allowedRetriveValue)
128	>	else :
129	>	files=[]
130	>	for arun in runselection:
131	>	try:
132	>	filesinrun = api.listFiles(path=self.datasetPath,retriveList=allowedRetriveValue,runNumber=arun)
133	>	files.extend(filesinrun)
134	>	except:
135	>	msg="WARNING: problem extracting info from DBS for run %s "%arun
136	>	common.logger.message(msg)
137	>	pass
138	>
139	>	except DbsBadRequest, msg:
140		raise DataDiscoveryError(msg)
141		except DBSError, msg:
142		raise DataDiscoveryError(msg)
143
144	<	if len(self.datasets) == 0:
145	<	raise DataDiscoveryError("DatasetPath=%s unknown to DBS" %self.datasetPath)
146	<	if len(self.datasets) > 1:
147	<	raise DataDiscoveryError("DatasetPath=%s is ambiguous" %self.datasetPath)
144	>	# parse files and fill arrays
145	>	for file in files :
146	>	parList = []
147	>	filename = file['LogicalFileName']
148	>	# asked retry the list of parent for the given child
149	>	if useParent: parList = [x['LogicalFileName'] for x in file['ParentList']]
150	>	self.parent[filename] = parList
151	>	if filename.find('.dat') < 0 :
152	>	fileblock = file['Block']['Name']
153	>	events    = file['NumberOfEvents']
154	>	# number of events per block
155	>	if fileblock in self.eventsPerBlock.keys() :
156	>	self.eventsPerBlock[fileblock] += events
157	>	else :
158	>	self.eventsPerBlock[fileblock] = events
159	>	# number of events per file
160	>	self.eventsPerFile[filename] = events
161	>
162	>	# number of events per block
163	>	if fileblock in self.blocksinfo.keys() :
164	>	self.blocksinfo[fileblock].append(filename)
165	>	else :
166	>	self.blocksinfo[fileblock] = [filename]
167
168	<	try:
169	<	self.dbsdataset = self.datasets[0].get('datasetPathName')
168	>	# total number of events
169	>	self.maxEvents += events
170	>
171	>	for block in self.eventsPerBlock.keys() :
172	>	common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(self.eventsPerBlock[block],block))
173
88	–	self.eventsPerBlock = dbs.getEventsPerBlock(self.dbsdataset)
89	–	self.blocksinfo = dbs.getDatasetFileBlocks(self.dbsdataset)
90	–	self.eventsPerFile = dbs.getEventsPerFile(self.dbsdataset)
91	–	except DBSError, ex:
92	–	raise DataDiscoveryError(ex.getErrorMessage())
93	–
174		if len(self.eventsPerBlock) <= 0:
175	<	raise NotExistingDatasetError (("\nNo data for %s in DBS\nPlease check"
175	>	raise NotExistingDatasetError(("\nNo data for %s in DBS\nPlease check"
176		+ " dataset path variables in crab.cfg")
177	<	% self.dbsdataset)
177	>	% self.datasetPath)
178
179
180		# #################################################
#	Line 102 | Line 182 | class DataDiscovery:
182		"""
183		max events
184		"""
185	<	## loop over the event collections
106	<	nevts=0
107	<	for evc_evts in self.eventsPerBlock.values():
108	<	nevts=nevts+evc_evts
109	<
110	<	return nevts
185	>	return self.maxEvents
186
187		# #################################################
188		def getEventsPerBlock(self):
#	Line 130 | Line 205 | class DataDiscovery:
205		"""
206		return self.blocksinfo
207
208	+	# #################################################
209	+	def getParent(self):
210	+	"""
211	+	return parent grouped by file
212	+	"""
213	+	return self.parent
214	+
215		########################################################################

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines