CRAB/python/DBSInfo.py

#!/usr/bin/env python
import sys, os, string, re, commands
import exceptions
import common
from crab_exceptions import *
try:
    import dbsCgiApi
    import dbsApi
except:
    try:
        Crabpydir=commands.getoutput('which crab')
        Topdir=string.replace(Crabpydir,'/python/crab','')
        sys.path.append(Topdir+'/DBSAPI')
        import dbsCgiApi
        import dbsApi
    except:
        msg="ERROR no DBS API available"
        raise CrabException(msg)

# #######################################
class DBSError(exceptions.Exception):
    def __init__(self, errorName, errorMessage):
        args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
        exceptions.Exception.__init__(self, args)
        pass

    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# #######################################
class DBSInvalidDataTierError(exceptions.Exception):
    def __init__(self, errorName, errorMessage):
        args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
        exceptions.Exception.__init__(self, args)
        pass
                                                                                      
    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
class DBSInfoError:
    def __init__(self, url):
        print '\nERROR accessing DBS url : '+url+'\n'
        pass

##################################################################################
# Class to extract info from DBS 
###############################################################################

class DBSInfo:
    def __init__(self):
        # Construct api object
        self.api = dbsCgiApi.DbsCgiApi()
        # Configure api logging level
        # self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_QUIET_)
        #if common.logger.debugLevel() >= 4:
        # self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_INFO_)
        #if common.logger.debugLevel() >= 10:          
        # self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_ALL_)

# ####################################
    def getMatchingDatasets (self, owner, dataset):
        """ Query DBS to get provenance """
        try:
            list = self.api.listProcessedDatasets("/%s/*/%s" % (dataset, owner))
        except dbsApi.InvalidDataTier, ex:
            raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
        except dbsApi.DbsApiException, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        except dbsCgiApi.DbsCgiToolError , ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        except dbsCgiApi.DbsCgiApiException , ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())

        return list

# ####################################
    def getDatasetProvenance(self, path, dataTiers):
        """
        query DBS to get provenance
        """
        try:
            datasetParentList = self.api.getDatasetProvenance(path,dataTiers)
        except dbsApi.InvalidDataTier, ex:
            raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())  
        except dbsApi.DbsApiException, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        except dbsCgiApi.DbsCgiApiException , ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        return datasetParentList                                                                                                            
        #parent = {}
        #for aparent in datasetParentList:
        #  common.logger.debug(6, "DBSInfo: parent path is "+aparent.getDatasetPath()+" datatier is "+aparent.getDataTier())
        #  parent[aparent.getDatasetPath()]=aparent.getDataTier()
        #
        #return parent

# ####################################
    def getDatasetContents(self, path):
        """
        query DBS to get event collections
        """
        try:
            fileBlockList = self.api.getDatasetContents(path)
        except dbsApi.DbsApiException, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        except dbsCgiApi.DbsCgiApiException , ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        ## get the fileblock and event collections
        nevtsbyblock= {}
        for fileBlock in fileBlockList:
            ## get the event collections for each block
            eventCollectionList = fileBlock.get('eventCollectionList')
            nevts=0
            for eventCollection in eventCollectionList:
                common.logger.debug(20,"DBSInfo:  evc: "+eventCollection.get('collectionName')+" nevts:%i"%eventCollection.get('numberOfEvents')) 
                nevts=nevts+eventCollection.get('numberOfEvents')
            common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(nevts,fileBlock.get('blockName')))
            nevtsbyblock[fileBlock.get('blockName')]=nevts

        # returning a map of fileblock-nevts  will be enough for now
        # TODO: in future the EvC collections grouped by fileblock should be returned
        
        return nevtsbyblock

Revision:	1.6
Committed:	Mon May 29 01:55:14 2006 UTC (18 years, 11 months ago) by afanfani
Content type:	text/x-python
Branch:	MAIN
Changes since 1.5:	+6 -0 lines
Log Message:	catch the DbsCgiApi Expection
#	User	Rev	Content
1	afanfani	1.1	#!/usr/bin/env python
2	afanfani	1.2	import sys, os, string, re, commands
3			import exceptions
4	afanfani	1.1	import common
5			from crab_exceptions import *
6			try:
7	slacapra	1.4	import dbsCgiApi
8			import dbsApi
9	afanfani	1.1	except:
10	slacapra	1.4	try:
11			Crabpydir=commands.getoutput('which crab')
12			Topdir=string.replace(Crabpydir,'/python/crab','')
13			sys.path.append(Topdir+'/DBSAPI')
14			import dbsCgiApi
15			import dbsApi
16			except:
17			msg="ERROR no DBS API available"
18			raise CrabException(msg)
19	afanfani	1.2
20			# #######################################
21			class DBSError(exceptions.Exception):
22	slacapra	1.4	def __init__(self, errorName, errorMessage):
23			args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
24			exceptions.Exception.__init__(self, args)
25			pass
26
27			def getErrorMessage(self):
28			""" Return exception error """
29			return "%s" % (self.args)
30	afanfani	1.2
31			# #######################################
32			class DBSInvalidDataTierError(exceptions.Exception):
33	slacapra	1.4	def __init__(self, errorName, errorMessage):
34			args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
35			exceptions.Exception.__init__(self, args)
36			pass
37	afanfani	1.2
38	slacapra	1.4	def getErrorMessage(self):
39			""" Return exception error """
40			return "%s" % (self.args)
41	afanfani	1.1
42	afanfani	1.2	# ####################################
43	afanfani	1.1	class DBSInfoError:
44	slacapra	1.4	def __init__(self, url):
45			print '\nERROR accessing DBS url : '+url+'\n'
46			pass
47	afanfani	1.1
48			##################################################################################
49			# Class to extract info from DBS
50			###############################################################################
51
52			class DBSInfo:
53	slacapra	1.4	def __init__(self):
54			# Construct api object
55			self.api = dbsCgiApi.DbsCgiApi()
56			# Configure api logging level
57	afanfani	1.5	# self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_QUIET_)
58	slacapra	1.4	#if common.logger.debugLevel() >= 4:
59			# self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_INFO_)
60			#if common.logger.debugLevel() >= 10:
61			# self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_ALL_)
62	afanfani	1.1
63			# ####################################
64	slacapra	1.4	def getMatchingDatasets (self, owner, dataset):
65			""" Query DBS to get provenance """
66			try:
67	afanfani	1.5	list = self.api.listProcessedDatasets("/%s/*/%s" % (dataset, owner))
68	slacapra	1.4	except dbsApi.InvalidDataTier, ex:
69			raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
70			except dbsApi.DbsApiException, ex:
71			raise DBSError(ex.getClassName(),ex.getErrorMessage())
72	afanfani	1.5	except dbsCgiApi.DbsCgiToolError , ex:
73			raise DBSError(ex.getClassName(),ex.getErrorMessage())
74	afanfani	1.6	except dbsCgiApi.DbsCgiApiException , ex:
75			raise DBSError(ex.getClassName(),ex.getErrorMessage())
76	afanfani	1.5
77	slacapra	1.4	return list
78	afanfani	1.3
79			# ####################################
80	slacapra	1.4	def getDatasetProvenance(self, path, dataTiers):
81			"""
82			query DBS to get provenance
83			"""
84			try:
85			datasetParentList = self.api.getDatasetProvenance(path,dataTiers)
86			except dbsApi.InvalidDataTier, ex:
87			raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
88			except dbsApi.DbsApiException, ex:
89			raise DBSError(ex.getClassName(),ex.getErrorMessage())
90	afanfani	1.6	except dbsCgiApi.DbsCgiApiException , ex:
91			raise DBSError(ex.getClassName(),ex.getErrorMessage())
92	slacapra	1.4	return datasetParentList
93			#parent = {}
94			#for aparent in datasetParentList:
95			# common.logger.debug(6, "DBSInfo: parent path is "+aparent.getDatasetPath()+" datatier is "+aparent.getDataTier())
96			# parent[aparent.getDatasetPath()]=aparent.getDataTier()
97			#
98			#return parent
99	afanfani	1.1
100			# ####################################
101	slacapra	1.4	def getDatasetContents(self, path):
102			"""
103			query DBS to get event collections
104			"""
105			try:
106			fileBlockList = self.api.getDatasetContents(path)
107			except dbsApi.DbsApiException, ex:
108			raise DBSError(ex.getClassName(),ex.getErrorMessage())
109	afanfani	1.6	except dbsCgiApi.DbsCgiApiException , ex:
110			raise DBSError(ex.getClassName(),ex.getErrorMessage())
111	slacapra	1.4	## get the fileblock and event collections
112			nevtsbyblock= {}
113			for fileBlock in fileBlockList:
114	afanfani	1.1	## get the event collections for each block
115	afanfani	1.5	eventCollectionList = fileBlock.get('eventCollectionList')
116	afanfani	1.1	nevts=0
117			for eventCollection in eventCollectionList:
118	afanfani	1.5	common.logger.debug(20,"DBSInfo: evc: "+eventCollection.get('collectionName')+" nevts:%i"%eventCollection.get('numberOfEvents'))
119			nevts=nevts+eventCollection.get('numberOfEvents')
120			common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(nevts,fileBlock.get('blockName')))
121			nevtsbyblock[fileBlock.get('blockName')]=nevts
122	afanfani	1.1
123	slacapra	1.4	# returning a map of fileblock-nevts will be enough for now
124			# TODO: in future the EvC collections grouped by fileblock should be returned
125
126			return nevtsbyblock
127	afanfani	1.1