CRAB/python/DBSInfo.py

#!/usr/bin/env python
import sys, os, string, re, commands
import exceptions
import common
from crab_exceptions import *
try:
    import dbsCgiApi
    import dbsApi
except:
    try:
        Crabpydir=commands.getoutput('which crab')
        Topdir=string.replace(Crabpydir,'/python/crab','')
        sys.path.append(Topdir+'/DBSAPI')
        import dbsCgiApi
        import dbsApi
    except:
        msg="ERROR no DBS API available"
        raise CrabException(msg)

# #######################################
class DBSError(exceptions.Exception):
    def __init__(self, errorName, errorMessage):
        args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
        exceptions.Exception.__init__(self, args)
        pass

    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# #######################################
class DBSInvalidDataTierError(exceptions.Exception):
    def __init__(self, errorName, errorMessage):
        args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
        exceptions.Exception.__init__(self, args)
        pass
                                                                                      
    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
class DBSInfoError:
    def __init__(self, url):
        print '\nERROR accessing DBS url : '+url+'\n'
        pass

##################################################################################
# Class to extract info from DBS 
###############################################################################

class DBSInfo:
    def __init__(self):
        # Construct api object
        self.api = dbsCgiApi.DbsCgiApi()
        # Configure api logging level
        # self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_QUIET_)
        #if common.logger.debugLevel() >= 4:
        # self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_INFO_)
        #if common.logger.debugLevel() >= 10:          
        # self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_ALL_)

# ####################################
    def getMatchingDatasets (self, owner, dataset):
        """ Query DBS to get provenance """
        try:
            list = self.api.listProcessedDatasets("/%s/*/%s" % (dataset, owner))
        except dbsApi.InvalidDataTier, ex:
            raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
        except dbsApi.DbsApiException, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        except dbsCgiApi.DbsCgiToolError , ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())

        return list

# ####################################
    def getDatasetProvenance(self, path, dataTiers):
        """
        query DBS to get provenance
        """
        try:
            datasetParentList = self.api.getDatasetProvenance(path,dataTiers)
        except dbsApi.InvalidDataTier, ex:
            raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())  
        except dbsApi.DbsApiException, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        return datasetParentList                                                                                                            
        #parent = {}
        #for aparent in datasetParentList:
        #  common.logger.debug(6, "DBSInfo: parent path is "+aparent.getDatasetPath()+" datatier is "+aparent.getDataTier())
        #  parent[aparent.getDatasetPath()]=aparent.getDataTier()
        #
        #return parent

# ####################################
    def getDatasetContents(self, path):
        """
        query DBS to get event collections
        """
        try:
            fileBlockList = self.api.getDatasetContents(path)
        except dbsApi.DbsApiException, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        ## get the fileblock and event collections
        nevtsbyblock= {}
        for fileBlock in fileBlockList:
            ## get the event collections for each block
            eventCollectionList = fileBlock.get('eventCollectionList')
            nevts=0
            for eventCollection in eventCollectionList:
                common.logger.debug(20,"DBSInfo:  evc: "+eventCollection.get('collectionName')+" nevts:%i"%eventCollection.get('numberOfEvents')) 
                nevts=nevts+eventCollection.get('numberOfEvents')
            common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(nevts,fileBlock.get('blockName')))
            nevtsbyblock[fileBlock.get('blockName')]=nevts

        # returning a map of fileblock-nevts  will be enough for now
        # TODO: in future the EvC collections grouped by fileblock should be returned
        
        return nevtsbyblock

Revision:	1.5
Committed:	Thu May 18 18:46:22 2006 UTC (18 years, 11 months ago) by afanfani
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	post_cmssw_integration_20060527, pre_cmssw_integration_20060527
Changes since 1.4:	+10 -7 lines
Log Message:	mods to use new DBS and DLS API
#	User	Rev	Content
1	afanfani	1.1	#!/usr/bin/env python
2	afanfani	1.2	import sys, os, string, re, commands
3			import exceptions
4	afanfani	1.1	import common
5			from crab_exceptions import *
6			try:
7	slacapra	1.4	import dbsCgiApi
8			import dbsApi
9	afanfani	1.1	except:
10	slacapra	1.4	try:
11			Crabpydir=commands.getoutput('which crab')
12			Topdir=string.replace(Crabpydir,'/python/crab','')
13			sys.path.append(Topdir+'/DBSAPI')
14			import dbsCgiApi
15			import dbsApi
16			except:
17			msg="ERROR no DBS API available"
18			raise CrabException(msg)
19	afanfani	1.2
20			# #######################################
21			class DBSError(exceptions.Exception):
22	slacapra	1.4	def __init__(self, errorName, errorMessage):
23			args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
24			exceptions.Exception.__init__(self, args)
25			pass
26
27			def getErrorMessage(self):
28			""" Return exception error """
29			return "%s" % (self.args)
30	afanfani	1.2
31			# #######################################
32			class DBSInvalidDataTierError(exceptions.Exception):
33	slacapra	1.4	def __init__(self, errorName, errorMessage):
34			args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
35			exceptions.Exception.__init__(self, args)
36			pass
37	afanfani	1.2
38	slacapra	1.4	def getErrorMessage(self):
39			""" Return exception error """
40			return "%s" % (self.args)
41	afanfani	1.1
42	afanfani	1.2	# ####################################
43	afanfani	1.1	class DBSInfoError:
44	slacapra	1.4	def __init__(self, url):
45			print '\nERROR accessing DBS url : '+url+'\n'
46			pass
47	afanfani	1.1
48			##################################################################################
49			# Class to extract info from DBS
50			###############################################################################
51
52			class DBSInfo:
53	slacapra	1.4	def __init__(self):
54			# Construct api object
55			self.api = dbsCgiApi.DbsCgiApi()
56			# Configure api logging level
57	afanfani	1.5	# self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_QUIET_)
58	slacapra	1.4	#if common.logger.debugLevel() >= 4:
59			# self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_INFO_)
60			#if common.logger.debugLevel() >= 10:
61			# self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_ALL_)
62	afanfani	1.1
63			# ####################################
64	slacapra	1.4	def getMatchingDatasets (self, owner, dataset):
65			""" Query DBS to get provenance """
66			try:
67	afanfani	1.5	list = self.api.listProcessedDatasets("/%s/*/%s" % (dataset, owner))
68	slacapra	1.4	except dbsApi.InvalidDataTier, ex:
69			raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
70			except dbsApi.DbsApiException, ex:
71			raise DBSError(ex.getClassName(),ex.getErrorMessage())
72	afanfani	1.5	except dbsCgiApi.DbsCgiToolError , ex:
73			raise DBSError(ex.getClassName(),ex.getErrorMessage())
74
75	slacapra	1.4	return list
76	afanfani	1.3
77			# ####################################
78	slacapra	1.4	def getDatasetProvenance(self, path, dataTiers):
79			"""
80			query DBS to get provenance
81			"""
82			try:
83			datasetParentList = self.api.getDatasetProvenance(path,dataTiers)
84			except dbsApi.InvalidDataTier, ex:
85			raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
86			except dbsApi.DbsApiException, ex:
87			raise DBSError(ex.getClassName(),ex.getErrorMessage())
88			return datasetParentList
89			#parent = {}
90			#for aparent in datasetParentList:
91			# common.logger.debug(6, "DBSInfo: parent path is "+aparent.getDatasetPath()+" datatier is "+aparent.getDataTier())
92			# parent[aparent.getDatasetPath()]=aparent.getDataTier()
93			#
94			#return parent
95	afanfani	1.1
96			# ####################################
97	slacapra	1.4	def getDatasetContents(self, path):
98			"""
99			query DBS to get event collections
100			"""
101			try:
102			fileBlockList = self.api.getDatasetContents(path)
103			except dbsApi.DbsApiException, ex:
104			raise DBSError(ex.getClassName(),ex.getErrorMessage())
105			## get the fileblock and event collections
106			nevtsbyblock= {}
107			for fileBlock in fileBlockList:
108	afanfani	1.1	## get the event collections for each block
109	afanfani	1.5	eventCollectionList = fileBlock.get('eventCollectionList')
110	afanfani	1.1	nevts=0
111			for eventCollection in eventCollectionList:
112	afanfani	1.5	common.logger.debug(20,"DBSInfo: evc: "+eventCollection.get('collectionName')+" nevts:%i"%eventCollection.get('numberOfEvents'))
113			nevts=nevts+eventCollection.get('numberOfEvents')
114			common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(nevts,fileBlock.get('blockName')))
115			nevtsbyblock[fileBlock.get('blockName')]=nevts
116	afanfani	1.1
117	slacapra	1.4	# returning a map of fileblock-nevts will be enough for now
118			# TODO: in future the EvC collections grouped by fileblock should be returned
119
120			return nevtsbyblock
121	afanfani	1.1