CRAB/python/DBSInfo_EDM.py

#!/usr/bin/env python
import sys, os, string, re, commands
import exceptions
import common
from crab_exceptions import *
try:
    import dbsCgiApi
    import dbsApi
except:
    try:
        Crabpydir=commands.getoutput('which crab')
        Topdir=string.replace(Crabpydir,'/python/crab','')
        sys.path.append(Topdir+'/DBSAPI')
        import dbsCgiApi
        import dbsApi
    except:
        msg="ERROR no DBS API available"
        raise CrabException(msg)
                                                                                              
## for python 2.2 add the pyexpat.so to PYTHONPATH
pythonV=sys.version.split(' ')[0]
if pythonV.find('2.2') >= 0 :
    Crabpydir=commands.getoutput('which crab')
    Topdir=string.replace(Crabpydir,'/python/crab','')
    extradir=Topdir+'/DLSAPI/extra'
    if sys.path.count(extradir) <= 0:
        if os.path.exists(extradir):
            sys.path.insert(0, extradir)

# #######################################
class DBSError(exceptions.Exception):
    def __init__(self, errorName, errorMessage):
        args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
        exceptions.Exception.__init__(self, args)
        pass
    
    def getErrorMessage(self):
        """ Return error message """
        return "%s" % (self.args)

# #######################################
class DBSInvalidDataTierError(exceptions.Exception):
    def __init__(self, errorName, errorMessage):
        args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
        exceptions.Exception.__init__(self, args)
        pass
    
    def getErrorMessage(self):
        """ Return error message """
        return "%s" % (self.args)

# #######################################
class DBSInfoError:
    def __init__(self, url):
        print '\nERROR accessing DBS url : '+url+'\n'
        pass

##################################################################################
# Class to extract info from DBS 
###############################################################################

class DBSInfo_EDM:
    def __init__(self, dbs_url, dbs_instance):
        """
        Construct api object.
        """
        ## cgi service API
        args = {}
        args['instance']=dbs_instance

        common.logger.debug(3,"Accessing DBS at: "+dbs_url+" "+dbs_instance)

        self.api = dbsCgiApi.DbsCgiApi(dbs_url, args)
        ## set log level
        # self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_INFO_)
        #self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_QUIET_)

    def getMatchingDatasets (self, datasetPath):
        """ Query DBS to get provenance """
        try:
            list = self.api.listProcessedDatasets("%s" %datasetPath)
        except dbsApi.InvalidDataTier, ex:
            raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
        except dbsApi.DbsApiException, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        except dbsCgiApi.DbsCgiToolError , ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        except dbsCgiApi.DbsCgiBadResponse , ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())

        return list


    def getDatasetProvenance(self, path, dataTiers):
        """ Query DBS to get provenance """
        try:
            datasetParentList = self.api.getDatasetProvenance(path,dataTiers)
        except dbsApi.InvalidDataTier, ex:
            raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
        except dbsApi.DbsApiException, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        return datasetParentList                                                                                                            

    def getDatasetContents(self, path):
        """ Query DBS to get event collections """
        # count events per block
        nevtsbyblock = {}
        #print "FileBlock :",str(self.api.getDatasetContents (path))
        try:
            for fileBlock in self.api.getDatasetContents (path):
                ## get the event collections for each block
                nevts = 0
                for evc in fileBlock.get('eventCollectionList'):
                    nevts = nevts + evc.get('numberOfEvents')
                    common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(nevts,fileBlock.get('blockName')))
                    #print "BlockName ",fileBlock.get('blockName')
                    ## SL temp hack to get rid of a mismatch between block names as returned by DBS
                    tmp = string.split(fileBlock.get('blockName'),"/")
                    if (len(tmp)==4): del tmp[2]
                    blockName=string.join(tmp,"/")
                    #print "TMP ",blockName
                    # end hack
                    
                    nevtsbyblock[blockName]=nevts
                pass
        except dbsApi.DbsApiException, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())

        # returning a map of fileblock-nevts  will be enough for now
        # TODO: in future the EvC collections grouped by fileblock should be returned
        return nevtsbyblock


    def getDatasetFileBlocks(self, path):
        """ Query DBS to get files/fileblocks """
        try:
            FilesbyBlock={}
            for fileBlock in self.api.getDatasetFileBlocks(path):
                blockname=fileBlock.get('blockName')
                filesinblock=[]
                for files in fileBlock.get('fileList'):
                    #print "  block %s has file %s"%(blockname,files.getLogicalFileName())
                    filesinblock.append(files.get('logicalFileName'))
                FilesbyBlock[blockname]=filesinblock
        except dbsApi.DbsApiException, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())

        return FilesbyBlock
Revision:	1.6
Committed:	Thu Aug 3 22:44:40 2006 UTC (18 years, 9 months ago) by gutsche
Content type:	text/x-python
Branch:	MAIN
Changes since 1.5:	+17 -4 lines
Log Message:	Commit for Malina: Handled block-splitting for CMSSW and CE->SE in .jdl files
#	User	Rev	Content
1	gutsche	1.1	#!/usr/bin/env python
2	afanfani	1.3	import sys, os, string, re, commands
3	gutsche	1.1	import exceptions
4	afanfani	1.4	import common
5			from crab_exceptions import *
6	gutsche	1.1	try:
7	afanfani	1.3	import dbsCgiApi
8			import dbsApi
9	gutsche	1.1	except:
10	afanfani	1.3	try:
11			Crabpydir=commands.getoutput('which crab')
12			Topdir=string.replace(Crabpydir,'/python/crab','')
13			sys.path.append(Topdir+'/DBSAPI')
14			import dbsCgiApi
15			import dbsApi
16			except:
17			msg="ERROR no DBS API available"
18			raise CrabException(msg)
19
20			## for python 2.2 add the pyexpat.so to PYTHONPATH
21			pythonV=sys.version.split(' ')[0]
22			if pythonV.find('2.2') >= 0 :
23	slacapra	1.5	Crabpydir=commands.getoutput('which crab')
24			Topdir=string.replace(Crabpydir,'/python/crab','')
25			extradir=Topdir+'/DLSAPI/extra'
26			if sys.path.count(extradir) <= 0:
27			if os.path.exists(extradir):
28			sys.path.insert(0, extradir)
29	gutsche	1.1
30			# #######################################
31			class DBSError(exceptions.Exception):
32	slacapra	1.5	def __init__(self, errorName, errorMessage):
33			args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
34			exceptions.Exception.__init__(self, args)
35			pass
36
37			def getErrorMessage(self):
38			""" Return error message """
39			return "%s" % (self.args)
40	gutsche	1.1
41			# #######################################
42			class DBSInvalidDataTierError(exceptions.Exception):
43	slacapra	1.5	def __init__(self, errorName, errorMessage):
44			args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
45			exceptions.Exception.__init__(self, args)
46			pass
47
48			def getErrorMessage(self):
49			""" Return error message """
50			return "%s" % (self.args)
51	gutsche	1.1
52			# #######################################
53			class DBSInfoError:
54	slacapra	1.5	def __init__(self, url):
55			print '\nERROR accessing DBS url : '+url+'\n'
56			pass
57	gutsche	1.1
58			##################################################################################
59			# Class to extract info from DBS
60			###############################################################################
61
62			class DBSInfo_EDM:
63	gutsche	1.6	def __init__(self, dbs_url, dbs_instance):
64	slacapra	1.5	"""
65			Construct api object.
66			"""
67			## cgi service API
68			args = {}
69			args['instance']=dbs_instance
70
71	gutsche	1.6	common.logger.debug(3,"Accessing DBS at: "+dbs_url+" "+dbs_instance)
72
73			self.api = dbsCgiApi.DbsCgiApi(dbs_url, args)
74	slacapra	1.5	## set log level
75			# self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_INFO_)
76			#self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_QUIET_)
77
78			def getMatchingDatasets (self, datasetPath):
79			""" Query DBS to get provenance """
80			try:
81			list = self.api.listProcessedDatasets("%s" %datasetPath)
82			except dbsApi.InvalidDataTier, ex:
83			raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
84			except dbsApi.DbsApiException, ex:
85			raise DBSError(ex.getClassName(),ex.getErrorMessage())
86			except dbsCgiApi.DbsCgiToolError , ex:
87			raise DBSError(ex.getClassName(),ex.getErrorMessage())
88	gutsche	1.6	except dbsCgiApi.DbsCgiBadResponse , ex:
89			raise DBSError(ex.getClassName(),ex.getErrorMessage())
90	slacapra	1.5
91			return list
92
93
94			def getDatasetProvenance(self, path, dataTiers):
95			""" Query DBS to get provenance """
96			try:
97			datasetParentList = self.api.getDatasetProvenance(path,dataTiers)
98			except dbsApi.InvalidDataTier, ex:
99			raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
100			except dbsApi.DbsApiException, ex:
101			raise DBSError(ex.getClassName(),ex.getErrorMessage())
102			return datasetParentList
103
104			def getDatasetContents(self, path):
105			""" Query DBS to get event collections """
106			# count events per block
107			nevtsbyblock = {}
108	gutsche	1.6	#print "FileBlock :",str(self.api.getDatasetContents (path))
109	slacapra	1.5	try:
110			for fileBlock in self.api.getDatasetContents (path):
111			## get the event collections for each block
112			nevts = 0
113			for evc in fileBlock.get('eventCollectionList'):
114			nevts = nevts + evc.get('numberOfEvents')
115			common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(nevts,fileBlock.get('blockName')))
116	gutsche	1.6	#print "BlockName ",fileBlock.get('blockName')
117			## SL temp hack to get rid of a mismatch between block names as returned by DBS
118			tmp = string.split(fileBlock.get('blockName'),"/")
119			if (len(tmp)==4): del tmp[2]
120			blockName=string.join(tmp,"/")
121			#print "TMP ",blockName
122			# end hack
123
124			nevtsbyblock[blockName]=nevts
125			pass
126	slacapra	1.5	except dbsApi.DbsApiException, ex:
127			raise DBSError(ex.getClassName(),ex.getErrorMessage())
128
129			# returning a map of fileblock-nevts will be enough for now
130			# TODO: in future the EvC collections grouped by fileblock should be returned
131			return nevtsbyblock
132
133
134			def getDatasetFileBlocks(self, path):
135			""" Query DBS to get files/fileblocks """
136			try:
137			FilesbyBlock={}
138			for fileBlock in self.api.getDatasetFileBlocks(path):
139			blockname=fileBlock.get('blockName')
140			filesinblock=[]
141			for files in fileBlock.get('fileList'):
142			#print " block %s has file %s"%(blockname,files.getLogicalFileName())
143			filesinblock.append(files.get('logicalFileName'))
144			FilesbyBlock[blockname]=filesinblock
145			except dbsApi.DbsApiException, ex:
146			raise DBSError(ex.getClassName(),ex.getErrorMessage())
147	gutsche	1.1
148	slacapra	1.5	return FilesbyBlock