CRAB/python/DBSInfo_EDM.py

#!/usr/bin/env python
import sys, os, string, re, commands
import exceptions
import common
from crab_exceptions import *
try:
    import dbsCgiApi
    import dbsApi
except:
    try:
        Crabpydir=commands.getoutput('which crab')
        Topdir=string.replace(Crabpydir,'/python/crab','')
        sys.path.append(Topdir+'/DBSAPI')
        import dbsCgiApi
        import dbsApi
    except:
        msg="ERROR no DBS API available"
        raise CrabException(msg)
                                                                                              
## for python 2.2 add the pyexpat.so to PYTHONPATH
pythonV=sys.version.split(' ')[0]
if pythonV.find('2.2') >= 0 :
    Crabpydir=commands.getoutput('which crab')
    Topdir=string.replace(Crabpydir,'/python/crab','')
    extradir=Topdir+'/DLSAPI/extra'
    if sys.path.count(extradir) <= 0:
        if os.path.exists(extradir):
            sys.path.insert(0, extradir)

# #######################################
class DBSError(exceptions.Exception):
    def __init__(self, errorName, errorMessage):
        args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
        exceptions.Exception.__init__(self, args)
        pass
    
    def getErrorMessage(self):
        """ Return error message """
        return "%s" % (self.args)

# #######################################
class DBSInvalidDataTierError(exceptions.Exception):
    def __init__(self, errorName, errorMessage):
        args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
        exceptions.Exception.__init__(self, args)
        pass
    
    def getErrorMessage(self):
        """ Return error message """
        return "%s" % (self.args)

# #######################################
class DBSInfoError:
    def __init__(self, url):
        print '\nERROR accessing DBS url : '+url+'\n'
        pass

##################################################################################
# Class to extract info from DBS 
###############################################################################

class DBSInfo_EDM:
    def __init__(self, dbs_url, dbs_instance):
        """
        Construct api object.
        """
        ## cgi service API
        args = {}
        args['instance']=dbs_instance

        common.logger.debug(3,"Accessing DBS at: "+dbs_url+" "+dbs_instance)

        self.api = dbsCgiApi.DbsCgiApi(dbs_url, args)
        ## set log level
        # self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_INFO_)
        #self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_QUIET_)

    def getMatchingDatasets (self, datasetPath):
        """ Query DBS to get provenance """
        try:
            list = self.api.listProcessedDatasets("%s" %datasetPath)
        except dbsApi.InvalidDataTier, ex:
            raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
        except dbsApi.DbsApiException, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        except dbsCgiApi.DbsCgiToolError , ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        except dbsCgiApi.DbsCgiBadResponse , ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())

        return list


    def getDatasetProvenance(self, path, dataTiers):
        """ Query DBS to get provenance """
        try:
            datasetParentList = self.api.getDatasetProvenance(path,dataTiers)
        except dbsApi.InvalidDataTier, ex:
            raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
        except dbsApi.DbsApiException, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        return datasetParentList                                                                                                            

    def getEventsPerBlock(self, path):
        """ Query DBS to get event collections """
        # count events per block
        nevtsbyblock = {}
        try:
            contents = self.api.getDatasetContents(path)
        except dbsApi.DbsApiException, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        except dbsCgiApi.DbsCgiBadResponse, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        for fileBlock in contents:
            ## get the event collections for each block
            nevts = 0
            eventCollectionList = fileBlock.get('eventCollectionList')
            for evc in eventCollectionList:
                nevts = nevts + evc.get('numberOfEvents')

            common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(nevts,fileBlock.get('blockName')))
            nevtsbyblock[fileBlock.get('blockName')]=nevts

        # returning a map of fileblock-nevts  will be enough for now
        # TODO: in future the EvC collections grouped by fileblock should be returned
        return nevtsbyblock

    def getEventsPerFile(self, path):
        """ Query DBS to get a dictionary of files:(events/file) """
        numEventsByFile = {}
        try:
            contents = self.api.getDatasetContents(path)
        except dbsApi.DbsApiException, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        for fileBlock in contents:
            numEvents = 0
            eventCollectionList = fileBlock.get('eventCollectionList')
            for evc in eventCollectionList:
                numEvents = evc.get('numberOfEvents')
                fileList = evc.get('fileList')
                # As of 2006-08-10, event collections contain only one file
                # => fileList contains only one dictionary
                if len(fileList)>1:
                    msg = "Event collection contains more than one file!  Exiting.\n"
                    msg = msg + "CRAB and DBS must be upgraded to handle event collections with multiple files.\n"
                    raise CrabException(msg)
                fileDict = fileList[0]
                fileName = fileDict.get('logicalFileName')
                numEventsByFile[fileName] = numEvents
        return numEventsByFile

    def getDatasetFileBlocks(self, path):
        """ Query DBS to get files/fileblocks """
        try:
            FilesbyBlock={}
            try:
                allBlocks = self.api.getDatasetFileBlocks(path)
            except dbsCgiApi.DbsCgiBadResponse, ex:
                raise DBSError(ex.getClassName(), ex.getErrorMessage())
            for fileBlock in allBlocks:
                blockname=fileBlock.get('blockName')
                filesinblock=[]
                for files in fileBlock.get('fileList'):
                    #print "  block %s has file %s"%(blockname,files.getLogicalFileName())
                    filesinblock.append(files.get('logicalFileName'))
                FilesbyBlock[blockname]=filesinblock
        except dbsApi.DbsApiException, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())

        return FilesbyBlock
Revision:	1.5.4.1
Committed:	Wed Sep 13 17:32:42 2006 UTC (18 years, 7 months ago) by spiga
Content type:	text/x-python
Branch:	CRAB_BOSS4_v1
Changes since 1.5:	+47 -12 lines
Log Message:	merged with the head
#	User	Rev	Content
1	gutsche	1.1	#!/usr/bin/env python
2	afanfani	1.3	import sys, os, string, re, commands
3	gutsche	1.1	import exceptions
4	afanfani	1.4	import common
5			from crab_exceptions import *
6	gutsche	1.1	try:
7	afanfani	1.3	import dbsCgiApi
8			import dbsApi
9	gutsche	1.1	except:
10	afanfani	1.3	try:
11			Crabpydir=commands.getoutput('which crab')
12			Topdir=string.replace(Crabpydir,'/python/crab','')
13			sys.path.append(Topdir+'/DBSAPI')
14			import dbsCgiApi
15			import dbsApi
16			except:
17			msg="ERROR no DBS API available"
18			raise CrabException(msg)
19
20			## for python 2.2 add the pyexpat.so to PYTHONPATH
21			pythonV=sys.version.split(' ')[0]
22			if pythonV.find('2.2') >= 0 :
23	slacapra	1.5	Crabpydir=commands.getoutput('which crab')
24			Topdir=string.replace(Crabpydir,'/python/crab','')
25			extradir=Topdir+'/DLSAPI/extra'
26			if sys.path.count(extradir) <= 0:
27			if os.path.exists(extradir):
28			sys.path.insert(0, extradir)
29	gutsche	1.1
30			# #######################################
31			class DBSError(exceptions.Exception):
32	slacapra	1.5	def __init__(self, errorName, errorMessage):
33			args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
34			exceptions.Exception.__init__(self, args)
35			pass
36
37			def getErrorMessage(self):
38			""" Return error message """
39			return "%s" % (self.args)
40	gutsche	1.1
41			# #######################################
42			class DBSInvalidDataTierError(exceptions.Exception):
43	slacapra	1.5	def __init__(self, errorName, errorMessage):
44			args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
45			exceptions.Exception.__init__(self, args)
46			pass
47
48			def getErrorMessage(self):
49			""" Return error message """
50			return "%s" % (self.args)
51	gutsche	1.1
52			# #######################################
53			class DBSInfoError:
54	slacapra	1.5	def __init__(self, url):
55			print '\nERROR accessing DBS url : '+url+'\n'
56			pass
57	gutsche	1.1
58			##################################################################################
59			# Class to extract info from DBS
60			###############################################################################
61
62			class DBSInfo_EDM:
63	spiga	1.5.4.1	def __init__(self, dbs_url, dbs_instance):
64	slacapra	1.5	"""
65			Construct api object.
66			"""
67			## cgi service API
68			args = {}
69			args['instance']=dbs_instance
70
71	spiga	1.5.4.1	common.logger.debug(3,"Accessing DBS at: "+dbs_url+" "+dbs_instance)
72
73			self.api = dbsCgiApi.DbsCgiApi(dbs_url, args)
74	slacapra	1.5	## set log level
75			# self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_INFO_)
76			#self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_QUIET_)
77
78			def getMatchingDatasets (self, datasetPath):
79			""" Query DBS to get provenance """
80			try:
81			list = self.api.listProcessedDatasets("%s" %datasetPath)
82			except dbsApi.InvalidDataTier, ex:
83			raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
84			except dbsApi.DbsApiException, ex:
85			raise DBSError(ex.getClassName(),ex.getErrorMessage())
86			except dbsCgiApi.DbsCgiToolError , ex:
87			raise DBSError(ex.getClassName(),ex.getErrorMessage())
88	spiga	1.5.4.1	except dbsCgiApi.DbsCgiBadResponse , ex:
89			raise DBSError(ex.getClassName(),ex.getErrorMessage())
90	slacapra	1.5
91			return list
92
93
94			def getDatasetProvenance(self, path, dataTiers):
95			""" Query DBS to get provenance """
96			try:
97			datasetParentList = self.api.getDatasetProvenance(path,dataTiers)
98			except dbsApi.InvalidDataTier, ex:
99			raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
100			except dbsApi.DbsApiException, ex:
101			raise DBSError(ex.getClassName(),ex.getErrorMessage())
102			return datasetParentList
103
104	spiga	1.5.4.1	def getEventsPerBlock(self, path):
105	slacapra	1.5	""" Query DBS to get event collections """
106			# count events per block
107			nevtsbyblock = {}
108			try:
109	spiga	1.5.4.1	contents = self.api.getDatasetContents(path)
110	slacapra	1.5	except dbsApi.DbsApiException, ex:
111			raise DBSError(ex.getClassName(),ex.getErrorMessage())
112	spiga	1.5.4.1	except dbsCgiApi.DbsCgiBadResponse, ex:
113			raise DBSError(ex.getClassName(),ex.getErrorMessage())
114			for fileBlock in contents:
115			## get the event collections for each block
116			nevts = 0
117			eventCollectionList = fileBlock.get('eventCollectionList')
118			for evc in eventCollectionList:
119			nevts = nevts + evc.get('numberOfEvents')
120
121			common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(nevts,fileBlock.get('blockName')))
122			nevtsbyblock[fileBlock.get('blockName')]=nevts
123	slacapra	1.5
124			# returning a map of fileblock-nevts will be enough for now
125			# TODO: in future the EvC collections grouped by fileblock should be returned
126			return nevtsbyblock
127
128	spiga	1.5.4.1	def getEventsPerFile(self, path):
129			""" Query DBS to get a dictionary of files:(events/file) """
130			numEventsByFile = {}
131			try:
132			contents = self.api.getDatasetContents(path)
133			except dbsApi.DbsApiException, ex:
134			raise DBSError(ex.getClassName(),ex.getErrorMessage())
135			for fileBlock in contents:
136			numEvents = 0
137			eventCollectionList = fileBlock.get('eventCollectionList')
138			for evc in eventCollectionList:
139			numEvents = evc.get('numberOfEvents')
140			fileList = evc.get('fileList')
141			# As of 2006-08-10, event collections contain only one file
142			# => fileList contains only one dictionary
143			if len(fileList)>1:
144			msg = "Event collection contains more than one file! Exiting.\n"
145			msg = msg + "CRAB and DBS must be upgraded to handle event collections with multiple files.\n"
146			raise CrabException(msg)
147			fileDict = fileList[0]
148			fileName = fileDict.get('logicalFileName')
149			numEventsByFile[fileName] = numEvents
150			return numEventsByFile
151	slacapra	1.5
152			def getDatasetFileBlocks(self, path):
153			""" Query DBS to get files/fileblocks """
154			try:
155			FilesbyBlock={}
156	spiga	1.5.4.1	try:
157			allBlocks = self.api.getDatasetFileBlocks(path)
158			except dbsCgiApi.DbsCgiBadResponse, ex:
159			raise DBSError(ex.getClassName(), ex.getErrorMessage())
160			for fileBlock in allBlocks:
161	slacapra	1.5	blockname=fileBlock.get('blockName')
162			filesinblock=[]
163			for files in fileBlock.get('fileList'):
164			#print " block %s has file %s"%(blockname,files.getLogicalFileName())
165			filesinblock.append(files.get('logicalFileName'))
166			FilesbyBlock[blockname]=filesinblock
167			except dbsApi.DbsApiException, ex:
168			raise DBSError(ex.getClassName(),ex.getErrorMessage())
169	gutsche	1.1
170	slacapra	1.5	return FilesbyBlock