CRAB/python/DBSInfo.py

#!/usr/bin/env python
import sys, os, string, re, commands
import exceptions
import common
from crab_exceptions import *
try:
    import dbsCgiApi
    import dbsApi
except:
    try:
        Crabpydir=commands.getoutput('which crab')
        Topdir=string.replace(Crabpydir,'/python/crab','')
        sys.path.append(Topdir+'/DBSAPI')
        import dbsCgiApi
        import dbsApi
    except:
        msg="ERROR no DBS API available"
        raise CrabException(msg)

## for python 2.2 add the pyexpat.so to PYTHONPATH
pythonV=sys.version.split(' ')[0]
if pythonV.find('2.2') >= 0 :
    Crabpydir=commands.getoutput('which crab')
    Topdir=string.replace(Crabpydir,'/python/crab','')
    extradir=Topdir+'/DLSAPI/extra'
    if sys.path.count(extradir) <= 0:
        if os.path.exists(extradir):
            sys.path.insert(0, extradir)

# #######################################
class DBSError(exceptions.Exception):
    def __init__(self, errorName, errorMessage):
        args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
        exceptions.Exception.__init__(self, args)
        pass

    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# #######################################
class DBSInvalidDataTierError(exceptions.Exception):
    def __init__(self, errorName, errorMessage):
        args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
        exceptions.Exception.__init__(self, args)
        pass
                                                                                      
    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
class DBSInfoError:
    def __init__(self, url):
        print '\nERROR accessing DBS url : '+url+'\n'
        pass

##################################################################################
# Class to extract info from DBS 
###############################################################################

class DBSInfo:
    def __init__(self):
        # Construct api object
        self.api = dbsCgiApi.DbsCgiApi()
        # Configure api logging level
        # self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_QUIET_)
        #if common.logger.debugLevel() >= 4:
        # self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_INFO_)
        #if common.logger.debugLevel() >= 10:          
        # self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_ALL_)

# ####################################
    def getMatchingDatasets (self, owner, dataset):
        """ Query DBS to get provenance """
        try:
            list = self.api.listProcessedDatasets("/%s/*/%s" % (dataset, owner))
        except dbsApi.InvalidDataTier, ex:
            raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
        except dbsApi.DbsApiException, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        except dbsCgiApi.DbsCgiToolError , ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        except dbsCgiApi.DbsCgiApiException , ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())

        return list

# ####################################
    def getDatasetProvenance(self, path, dataTiers):
        """
        query DBS to get provenance
        """
        try:
            datasetParentList = self.api.getDatasetProvenance(path,dataTiers)
        except dbsApi.InvalidDataTier, ex:
            raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())  
        except dbsApi.DbsApiException, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        except dbsCgiApi.DbsCgiApiException , ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        return datasetParentList                                                                                                            
        #parent = {}
        #for aparent in datasetParentList:
        #  common.logger.debug(6, "DBSInfo: parent path is "+aparent.getDatasetPath()+" datatier is "+aparent.getDataTier())
        #  parent[aparent.getDatasetPath()]=aparent.getDataTier()
        #
        #return parent

# ####################################
    def getDatasetContents(self, path):
        """
        query DBS to get event collections
        """
        try:
            fileBlockList = self.api.getDatasetContents(path)
        except dbsApi.DbsApiException, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        except dbsCgiApi.DbsCgiApiException , ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        ## get the fileblock and event collections
        nevtsbyblock= {}
        for fileBlock in fileBlockList:
            ## get the event collections for each block
            eventCollectionList = fileBlock.get('eventCollectionList')
            nevts=0
            for eventCollection in eventCollectionList:
                common.logger.debug(20,"DBSInfo:  evc: "+eventCollection.get('collectionName')+" nevts:%i"%eventCollection.get('numberOfEvents')) 
                nevts=nevts+eventCollection.get('numberOfEvents')
            common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(nevts,fileBlock.get('blockName')))
            nevtsbyblock[fileBlock.get('blockName')]=nevts

        # returning a map of fileblock-nevts  will be enough for now
        # TODO: in future the EvC collections grouped by fileblock should be returned
        
        return nevtsbyblock

Revision:	1.9
Committed:	Thu Jul 6 13:28:15 2006 UTC (18 years, 9 months ago) by slacapra
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_1_2_1, CRAB_1_2_0
Branch point for:	CRAB_BOSS4_v1, CRAB_BOSS4
Changes since 1.8:	+0 -1 lines
Log Message:	forgot a print
#	User	Rev	Content
1	afanfani	1.1	#!/usr/bin/env python
2	afanfani	1.2	import sys, os, string, re, commands
3			import exceptions
4	afanfani	1.1	import common
5			from crab_exceptions import *
6			try:
7	slacapra	1.4	import dbsCgiApi
8			import dbsApi
9	afanfani	1.1	except:
10	slacapra	1.4	try:
11			Crabpydir=commands.getoutput('which crab')
12			Topdir=string.replace(Crabpydir,'/python/crab','')
13			sys.path.append(Topdir+'/DBSAPI')
14			import dbsCgiApi
15			import dbsApi
16			except:
17			msg="ERROR no DBS API available"
18			raise CrabException(msg)
19	afanfani	1.2
20	afanfani	1.7	## for python 2.2 add the pyexpat.so to PYTHONPATH
21			pythonV=sys.version.split(' ')[0]
22			if pythonV.find('2.2') >= 0 :
23	slacapra	1.8	Crabpydir=commands.getoutput('which crab')
24			Topdir=string.replace(Crabpydir,'/python/crab','')
25			extradir=Topdir+'/DLSAPI/extra'
26			if sys.path.count(extradir) <= 0:
27			if os.path.exists(extradir):
28			sys.path.insert(0, extradir)
29	afanfani	1.7
30	afanfani	1.2	# #######################################
31			class DBSError(exceptions.Exception):
32	slacapra	1.4	def __init__(self, errorName, errorMessage):
33			args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
34			exceptions.Exception.__init__(self, args)
35			pass
36
37			def getErrorMessage(self):
38			""" Return exception error """
39			return "%s" % (self.args)
40	afanfani	1.2
41			# #######################################
42			class DBSInvalidDataTierError(exceptions.Exception):
43	slacapra	1.4	def __init__(self, errorName, errorMessage):
44			args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
45			exceptions.Exception.__init__(self, args)
46			pass
47	afanfani	1.2
48	slacapra	1.4	def getErrorMessage(self):
49			""" Return exception error """
50			return "%s" % (self.args)
51	afanfani	1.1
52	afanfani	1.2	# ####################################
53	afanfani	1.1	class DBSInfoError:
54	slacapra	1.4	def __init__(self, url):
55			print '\nERROR accessing DBS url : '+url+'\n'
56			pass
57	afanfani	1.1
58			##################################################################################
59			# Class to extract info from DBS
60			###############################################################################
61
62			class DBSInfo:
63	slacapra	1.4	def __init__(self):
64			# Construct api object
65			self.api = dbsCgiApi.DbsCgiApi()
66			# Configure api logging level
67	afanfani	1.5	# self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_QUIET_)
68	slacapra	1.4	#if common.logger.debugLevel() >= 4:
69			# self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_INFO_)
70			#if common.logger.debugLevel() >= 10:
71			# self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_ALL_)
72	afanfani	1.1
73			# ####################################
74	slacapra	1.4	def getMatchingDatasets (self, owner, dataset):
75			""" Query DBS to get provenance """
76			try:
77	afanfani	1.5	list = self.api.listProcessedDatasets("/%s/*/%s" % (dataset, owner))
78	slacapra	1.4	except dbsApi.InvalidDataTier, ex:
79			raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
80			except dbsApi.DbsApiException, ex:
81			raise DBSError(ex.getClassName(),ex.getErrorMessage())
82	afanfani	1.5	except dbsCgiApi.DbsCgiToolError , ex:
83			raise DBSError(ex.getClassName(),ex.getErrorMessage())
84	afanfani	1.6	except dbsCgiApi.DbsCgiApiException , ex:
85			raise DBSError(ex.getClassName(),ex.getErrorMessage())
86	afanfani	1.5
87	slacapra	1.4	return list
88	afanfani	1.3
89			# ####################################
90	slacapra	1.4	def getDatasetProvenance(self, path, dataTiers):
91			"""
92			query DBS to get provenance
93			"""
94			try:
95			datasetParentList = self.api.getDatasetProvenance(path,dataTiers)
96			except dbsApi.InvalidDataTier, ex:
97			raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
98			except dbsApi.DbsApiException, ex:
99			raise DBSError(ex.getClassName(),ex.getErrorMessage())
100	afanfani	1.6	except dbsCgiApi.DbsCgiApiException , ex:
101			raise DBSError(ex.getClassName(),ex.getErrorMessage())
102	slacapra	1.4	return datasetParentList
103			#parent = {}
104			#for aparent in datasetParentList:
105			# common.logger.debug(6, "DBSInfo: parent path is "+aparent.getDatasetPath()+" datatier is "+aparent.getDataTier())
106			# parent[aparent.getDatasetPath()]=aparent.getDataTier()
107			#
108			#return parent
109	afanfani	1.1
110			# ####################################
111	slacapra	1.4	def getDatasetContents(self, path):
112			"""
113			query DBS to get event collections
114			"""
115			try:
116			fileBlockList = self.api.getDatasetContents(path)
117			except dbsApi.DbsApiException, ex:
118			raise DBSError(ex.getClassName(),ex.getErrorMessage())
119	afanfani	1.6	except dbsCgiApi.DbsCgiApiException , ex:
120			raise DBSError(ex.getClassName(),ex.getErrorMessage())
121	slacapra	1.4	## get the fileblock and event collections
122			nevtsbyblock= {}
123			for fileBlock in fileBlockList:
124	afanfani	1.1	## get the event collections for each block
125	afanfani	1.5	eventCollectionList = fileBlock.get('eventCollectionList')
126	afanfani	1.1	nevts=0
127			for eventCollection in eventCollectionList:
128	afanfani	1.5	common.logger.debug(20,"DBSInfo: evc: "+eventCollection.get('collectionName')+" nevts:%i"%eventCollection.get('numberOfEvents'))
129			nevts=nevts+eventCollection.get('numberOfEvents')
130			common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(nevts,fileBlock.get('blockName')))
131			nevtsbyblock[fileBlock.get('blockName')]=nevts
132	afanfani	1.1
133	slacapra	1.4	# returning a map of fileblock-nevts will be enough for now
134			# TODO: in future the EvC collections grouped by fileblock should be returned
135
136			return nevtsbyblock
137	afanfani	1.1