ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/DataDiscovery.py

Comparing COMP/CRAB/python/DataDiscovery.py (file contents):
Revision 1.13 by slacapra, Mon Dec 18 12:28:06 2006 UTC vs.
Revision 1.19 by slacapra, Fri Jan 4 17:30:56 2008 UTC

#	Line 1 | Line 1
1		#!/usr/bin/env python
2	<	import sys, os, string, re
3	<	from DBSInfo import *
2	>	import exceptions
3	>	import DBSAPI.dbsApi
4	>	from DBSAPI.dbsApiException import *
5	>	import common
6	>	from crab_util import *
7
8
9	+	# #######################################
10	+	class DBSError(exceptions.Exception):
11	+	def __init__(self, errorName, errorMessage):
12	+	args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
13	+	exceptions.Exception.__init__(self, args)
14	+	pass
15	+
16	+	def getErrorMessage(self):
17	+	""" Return error message """
18	+	return "%s" % (self.args)
19	+
20	+	# #######################################
21	+	class DBSInvalidDataTierError(exceptions.Exception):
22	+	def __init__(self, errorName, errorMessage):
23	+	args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
24	+	exceptions.Exception.__init__(self, args)
25	+	pass
26	+
27	+	def getErrorMessage(self):
28	+	""" Return error message """
29	+	return "%s" % (self.args)
30	+
31	+	# #######################################
32	+	class DBSInfoError:
33	+	def __init__(self, url):
34	+	print '\nERROR accessing DBS url : '+url+'\n'
35	+	pass
36	+
37		# ####################################
38		class DataDiscoveryError(exceptions.Exception):
39		def __init__(self, errorMessage):
40	<	args=errorMessage
41	<	exceptions.Exception.__init__(self, args)
40	>	self.args=errorMessage
41	>	exceptions.Exception.__init__(self, self.args)
42		pass
43
44		def getErrorMessage(self):
#	Line 17 | Line 48 | class DataDiscoveryError(exceptions.Exce
48		# ####################################
49		class NotExistingDatasetError(exceptions.Exception):
50		def __init__(self, errorMessage):
51	<	args=errorMessage
52	<	exceptions.Exception.__init__(self, args)
51	>	self.args=errorMessage
52	>	exceptions.Exception.__init__(self, self.args)
53		pass
54
55		def getErrorMessage(self):
#	Line 28 | Line 59 | class NotExistingDatasetError(exceptions
59		# ####################################
60		class NoDataTierinProvenanceError(exceptions.Exception):
61		def __init__(self, errorMessage):
62	<	args=errorMessage
63	<	exceptions.Exception.__init__(self, args)
62	>	self.args=errorMessage
63	>	exceptions.Exception.__init__(self, self.args)
64		pass
65
66		def getErrorMessage(self):
#	Line 39 | Line 70 | class NoDataTierinProvenanceError(except
70		# ####################################
71		# class to find and extact info from published data
72		class DataDiscovery:
73	<	def __init__(self, datasetPath, dataTiers, cfg_params):
73	>	def __init__(self, datasetPath, cfg_params):
74
75	<	#       Attributes
75	>	#       Attributes
76		self.datasetPath = datasetPath
46	–	self.dataTiers = dataTiers
77		self.cfg_params = cfg_params
78
79		self.eventsPerBlock = {}  # DBS output: map fileblocks-events for collection
80		self.eventsPerFile = {}   # DBS output: map files-events
81	<	self.blocksinfo = {}  # DBS output: map fileblocks-files
82	<	#DBS output: max events computed by method getMaxEvents
81	>	self.blocksinfo = {}      # DBS output: map fileblocks-files
82	>	self.maxEvents = 0        # DBS output: max events
83
84		# ####################################
85		def fetchDBSInfo(self):
#	Line 58 | Line 88 | class DataDiscovery:
88		"""
89
90		## get DBS URL
91	<	try:
91	>	dbs_url="http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
92	>	if (self.cfg_params.has_key('CMSSW.dbs_url')):
93		dbs_url=self.cfg_params['CMSSW.dbs_url']
63	–	except KeyError:
64	–	dbs_url="http://cmsdoc.cern.ch/cms/test/aprom/DBS/CGIServer/prodquery"
94
95	<	## get info about the requested dataset
96	<	try:
97	<	dbs_instance=self.cfg_params['CMSSW.dbs_instance']
98	<	except KeyError:
99	<	dbs_instance="MCGlobal/Writer"
100	<
101	<	dbs = DBSInfo(dbs_url, dbs_instance)
95	>	common.logger.debug(3,"Accessing DBS at: "+dbs_url)
96	>
97	>	## check if runs are selected
98	>	runselection = []
99	>	if (self.cfg_params.has_key('CMSSW.runselection')):
100	>	runselection = parseRange2(self.cfg_params['CMSSW.runselection'])
101	>
102	>	## service API
103	>	args = {}
104	>	args['url']     = dbs_url
105	>	args['level']   = 'CRITICAL'
106	>
107	>	api = DBSAPI.dbsApi.DbsApi(args)
108		try:
109	<	self.datasets = dbs.getMatchingDatasets(self.datasetPath)
110	<	except dbsCgiApi.DbsCgiExecutionError, msg:
109	>	if len(runselection) <= 0 :
110	>	files = api.listDatasetFiles(self.datasetPath)
111	>	else :
112	>	files = api.listFiles(path=self.datasetPath, details=True)
113	>	except DbsBadRequest, msg:
114		raise DataDiscoveryError(msg)
115		except DBSError, msg:
116		raise DataDiscoveryError(msg)
117
118	<	if len(self.datasets) == 0:
119	<	raise DataDiscoveryError("DatasetPath=%s unknown to DBS" %self.datasetPath)
120	<	if len(self.datasets) > 1:
121	<	raise DataDiscoveryError("DatasetPath=%s is ambiguous" %self.datasetPath)
118	>	# parse files and fill arrays
119	>	for file in files :
120	>	filename = file['LogicalFileName']
121	>	if filename.find('.dat') < 0 :
122	>	fileblock = file['Block']['Name']
123	>	events    = file['NumberOfEvents']
124	>	continue_flag = 0
125	>	if len(runselection) > 0 :
126	>	runslist = file['RunsList']
127	>	for run in runslist :
128	>	runnumber = run['RunNumber']
129	>	for selected_run in runselection :
130	>	if runnumber == selected_run :
131	>	continue_flag = 1
132	>	else :
133	>	continue_flag = 1
134	>
135	>	if continue_flag == 1 :
136	>	# number of events per block
137	>	if fileblock in self.eventsPerBlock.keys() :
138	>	self.eventsPerBlock[fileblock] += events
139	>	else :
140	>	self.eventsPerBlock[fileblock] = events
141	>
142	>	# number of events per file
143	>	self.eventsPerFile[filename] = events
144	>
145	>	# number of events per block
146	>	if fileblock in self.blocksinfo.keys() :
147	>	self.blocksinfo[fileblock].append(filename)
148	>	else :
149	>	self.blocksinfo[fileblock] = [filename]
150
151	<	try:
152	<	self.dbsdataset = self.datasets[0].get('datasetPathName')
151	>	# total number of events
152	>	self.maxEvents += events
153	>
154	>	for block in self.eventsPerBlock.keys() :
155	>	common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(self.eventsPerBlock[block],block))
156
88	–	self.eventsPerBlock = dbs.getEventsPerBlock(self.dbsdataset)
89	–	self.blocksinfo = dbs.getDatasetFileBlocks(self.dbsdataset)
90	–	self.eventsPerFile = dbs.getEventsPerFile(self.dbsdataset)
91	–	except DBSError, ex:
92	–	raise DataDiscoveryError(ex.getErrorMessage())
93	–
157		if len(self.eventsPerBlock) <= 0:
158	<	raise NotExistingDatasetError (("\nNo data for %s in DBS\nPlease check"
158	>	raise NotExistingDatasetError(("\nNo data for %s in DBS\nPlease check"
159		+ " dataset path variables in crab.cfg")
160	<	% self.dbsdataset)
160	>	% self.datasetPath)
161
162
163		# #################################################
#	Line 102 | Line 165 | class DataDiscovery:
165		"""
166		max events
167		"""
168	<	## loop over the event collections
106	<	nevts=0
107	<	for evc_evts in self.eventsPerBlock.values():
108	<	nevts=nevts+evc_evts
109	<
110	<	return nevts
168	>	return self.maxEvents
169
170		# #################################################
171		def getEventsPerBlock(self):

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines