ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/DBSInfo_EDM.py

Comparing COMP/CRAB/python/DBSInfo_EDM.py (file contents):
Revision 1.4 by afanfani, Thu Jun 1 11:12:34 2006 UTC vs.
Revision 1.9 by slacapra, Wed Aug 16 15:43:46 2006 UTC

#	Line 20 | Line 20 | except:
20		## for python 2.2 add the pyexpat.so to PYTHONPATH
21		pythonV=sys.version.split(' ')[0]
22		if pythonV.find('2.2') >= 0 :
23	<	Crabpydir=commands.getoutput('which crab')
24	<	Topdir=string.replace(Crabpydir,'/python/crab','')
25	<	extradir=Topdir+'/DLSAPI/extra'
26	<	if sys.path.count(extradir) <= 0:
27	<	if os.path.exists(extradir):
28	<	sys.path.insert(0, extradir)
23	>	Crabpydir=commands.getoutput('which crab')
24	>	Topdir=string.replace(Crabpydir,'/python/crab','')
25	>	extradir=Topdir+'/DLSAPI/extra'
26	>	if sys.path.count(extradir) <= 0:
27	>	if os.path.exists(extradir):
28	>	sys.path.insert(0, extradir)
29
30		# #######################################
31		class DBSError(exceptions.Exception):
32	<	def __init__(self, errorName, errorMessage):
33	<	args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
34	<	exceptions.Exception.__init__(self, args)
35	<	pass
36	<
37	<	def getErrorMessage(self):
38	<	""" Return error message """
39	<	return "%s" % (self.args)
32	>	def __init__(self, errorName, errorMessage):
33	>	args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
34	>	exceptions.Exception.__init__(self, args)
35	>	pass
36	>
37	>	def getErrorMessage(self):
38	>	""" Return error message """
39	>	return "%s" % (self.args)
40
41		# #######################################
42		class DBSInvalidDataTierError(exceptions.Exception):
43	<	def __init__(self, errorName, errorMessage):
44	<	args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
45	<	exceptions.Exception.__init__(self, args)
46	<	pass
47	<
48	<	def getErrorMessage(self):
49	<	""" Return error message """
50	<	return "%s" % (self.args)
43	>	def __init__(self, errorName, errorMessage):
44	>	args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
45	>	exceptions.Exception.__init__(self, args)
46	>	pass
47	>
48	>	def getErrorMessage(self):
49	>	""" Return error message """
50	>	return "%s" % (self.args)
51
52		# #######################################
53		class DBSInfoError:
54	<	def __init__(self, url):
55	<	print '\nERROR accessing DBS url : '+url+'\n'
56	<	pass
54	>	def __init__(self, url):
55	>	print '\nERROR accessing DBS url : '+url+'\n'
56	>	pass
57
58		##################################################################################
59		# Class to extract info from DBS
60		###############################################################################
61
62		class DBSInfo_EDM:
63	<	def __init__(self, dbs_instance):
64	<	"""
65	<	Construct api object.
66	<	"""
67	<	## cgi service API
68	<	DEFAULT_URL = "http://cmsdoc.cern.ch/cms/aprom/DBS/CGIServer/prodquery"
69	<	args = {}
70	<	args['instance']=dbs_instance
71	<
72	<	self.api = dbsCgiApi.DbsCgiApi(DEFAULT_URL, args)
73	<	## set log level
74	<	# self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_INFO_)
75	<	#self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_QUIET_)
76	<
77	<	def getMatchingDatasets (self, datasetPath):
78	<	""" Query DBS to get provenance """
79	<	try:
80	<	list = self.api.listProcessedDatasets("%s" %datasetPath)
81	<	except dbsApi.InvalidDataTier, ex:
82	<	raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
83	<	except dbsApi.DbsApiException, ex:
84	<	raise DBSError(ex.getClassName(),ex.getErrorMessage())
85	<	except dbsCgiApi.DbsCgiToolError , ex:
86	<	raise DBSError(ex.getClassName(),ex.getErrorMessage())
87	<
88	<	return list
89	<
90	<
91	<	def getDatasetProvenance(self, path, dataTiers):
92	<	""" Query DBS to get provenance """
93	<	try:
94	<	datasetParentList = self.api.getDatasetProvenance(path,dataTiers)
95	<	except dbsApi.InvalidDataTier, ex:
96	<	raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
97	<	except dbsApi.DbsApiException, ex:
98	<	raise DBSError(ex.getClassName(),ex.getErrorMessage())
99	<	return datasetParentList
100	<
101	<	def getDatasetContents(self, path):
102	<	""" Query DBS to get event collections """
103	<	# count events per block
104	<	nevtsbyblock = {}
105	<	try:
106	<	for fileBlock in self.api.getDatasetContents (path):
107	<	## get the event collections for each block
108	<	nevts = 0
109	<	for evc in fileBlock.get('eventCollectionList'):
110	<	nevts = nevts + evc.get('numberOfEvents')
111	<	common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(nevts,fileBlock.get('blockName')))
112	<	nevtsbyblock[fileBlock.get('blockName')]=nevts
113	<	except dbsApi.DbsApiException, ex:
114	<	raise DBSError(ex.getClassName(),ex.getErrorMessage())
115	<
116	<	# returning a map of fileblock-nevts  will be enough for now
117	<	# TODO: in future the EvC collections grouped by fileblock should be returned
118	<	return nevtsbyblock
119	<
120	<
121	<	def getDatasetFileBlocks(self, path):
122	<	""" Query DBS to get files/fileblocks """
123	<	try:
124	<	FilesbyBlock={}
125	<	for fileBlock in self.api.getDatasetFileBlocks(path):
126	<	blockname=fileBlock.get('blockName')
127	<	filesinblock=[]
128	<	for files in fileBlock.get('fileList'):
129	<	#print "  block %s has file %s"%(blockname,files.getLogicalFileName())
130	<	filesinblock.append(files.get('logicalFileName'))
131	<	FilesbyBlock[blockname]=filesinblock
132	<	except dbsApi.DbsApiException, ex:
133	<	raise DBSError(ex.getClassName(),ex.getErrorMessage())
134	<
135	<	return FilesbyBlock
63	>	def __init__(self, dbs_url, dbs_instance):
64	>	"""
65	>	Construct api object.
66	>	"""
67	>	## cgi service API
68	>	args = {}
69	>	args['instance']=dbs_instance
70	>
71	>	common.logger.debug(3,"Accessing DBS at: "+dbs_url+" "+dbs_instance)
72	>
73	>	self.api = dbsCgiApi.DbsCgiApi(dbs_url, args)
74	>	## set log level
75	>	# self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_INFO_)
76	>	#self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_QUIET_)
77	>
78	>	def getMatchingDatasets (self, datasetPath):
79	>	""" Query DBS to get provenance """
80	>	try:
81	>	list = self.api.listProcessedDatasets("%s" %datasetPath)
82	>	except dbsApi.InvalidDataTier, ex:
83	>	raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
84	>	except dbsApi.DbsApiException, ex:
85	>	raise DBSError(ex.getClassName(),ex.getErrorMessage())
86	>	except dbsCgiApi.DbsCgiToolError , ex:
87	>	raise DBSError(ex.getClassName(),ex.getErrorMessage())
88	>	except dbsCgiApi.DbsCgiBadResponse , ex:
89	>	raise DBSError(ex.getClassName(),ex.getErrorMessage())
90	>
91	>	return list
92	>
93	>
94	>	def getDatasetProvenance(self, path, dataTiers):
95	>	""" Query DBS to get provenance """
96	>	try:
97	>	datasetParentList = self.api.getDatasetProvenance(path,dataTiers)
98	>	except dbsApi.InvalidDataTier, ex:
99	>	raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
100	>	except dbsApi.DbsApiException, ex:
101	>	raise DBSError(ex.getClassName(),ex.getErrorMessage())
102	>	return datasetParentList
103	>
104	>	def getEventsPerBlock(self, path):
105	>	""" Query DBS to get event collections """
106	>	# count events per block
107	>	nevtsbyblock = {}
108	>	try:
109	>	contents = self.api.getDatasetContents(path)
110	>	except dbsApi.DbsApiException, ex:
111	>	raise DBSError(ex.getClassName(),ex.getErrorMessage())
112	>	except dbsCgiApi.DbsCgiBadResponse, ex:
113	>	raise DBSError(ex.getClassName(),ex.getErrorMessage())
114	>	for fileBlock in contents:
115	>	## get the event collections for each block
116	>	nevts = 0
117	>	eventCollectionList = fileBlock.get('eventCollectionList')
118	>	for evc in eventCollectionList:
119	>	nevts = nevts + evc.get('numberOfEvents')
120	>
121	>	common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(nevts,fileBlock.get('blockName')))
122	>	nevtsbyblock[fileBlock.get('blockName')]=nevts
123	>
124	>	# returning a map of fileblock-nevts  will be enough for now
125	>	# TODO: in future the EvC collections grouped by fileblock should be returned
126	>	return nevtsbyblock
127	>
128	>	def getEventsPerFile(self, path):
129	>	""" Query DBS to get a dictionary of files:(events/file) """
130	>	numEventsByFile = {}
131	>	try:
132	>	contents = self.api.getDatasetContents(path)
133	>	except dbsApi.DbsApiException, ex:
134	>	raise DBSError(ex.getClassName(),ex.getErrorMessage())
135	>	for fileBlock in contents:
136	>	numEvents = 0
137	>	eventCollectionList = fileBlock.get('eventCollectionList')
138	>	for evc in eventCollectionList:
139	>	numEvents = evc.get('numberOfEvents')
140	>	fileList = evc.get('fileList')
141	>	# As of 2006-08-10, event collections contain only one file
142	>	# => fileList contains only one dictionary
143	>	if len(fileList)>1:
144	>	msg = "Event collection contains more than one file!  Exiting.\n"
145	>	msg = msg + "CRAB and DBS must be upgraded to handle event collections with multiple files.\n"
146	>	raise CrabException(msg)
147	>	fileDict = fileList[0]
148	>	fileName = fileDict.get('logicalFileName')
149	>	numEventsByFile[fileName] = numEvents
150	>	return numEventsByFile
151	>
152	>	def getDatasetFileBlocks(self, path):
153	>	""" Query DBS to get files/fileblocks """
154	>	try:
155	>	FilesbyBlock={}
156	>	try:
157	>	allBlocks = self.api.getDatasetFileBlocks(path)
158	>	except dbsCgiApi.DbsCgiBadResponse, ex:
159	>	raise DBSError(ex.getClassName(), ex.getErrorMessage())
160	>	for fileBlock in allBlocks:
161	>	blockname=fileBlock.get('blockName')
162	>	filesinblock=[]
163	>	for files in fileBlock.get('fileList'):
164	>	#print "  block %s has file %s"%(blockname,files.getLogicalFileName())
165	>	filesinblock.append(files.get('logicalFileName'))
166	>	FilesbyBlock[blockname]=filesinblock
167	>	except dbsApi.DbsApiException, ex:
168	>	raise DBSError(ex.getClassName(),ex.getErrorMessage())
169
170	+	return FilesbyBlock

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines