CRAB/python/DBSInfo.py

#!/usr/bin/env python
import sys, os, string, re, commands
import exceptions
import common
from crab_exceptions import *
try:
    import dbsCgiApi
    import dbsApi
except:
    try:
        Crabpydir=commands.getoutput('which crab')
        Topdir=string.replace(Crabpydir,'/python/crab','')
        sys.path.append(Topdir+'/DBSAPI')
        import dbsCgiApi
        import dbsApi
    except:
        msg="ERROR no DBS API available"
        raise CrabException(msg)
                                                                                              
## for python 2.2 add the pyexpat.so to PYTHONPATH
pythonV=sys.version.split(' ')[0]
if pythonV.find('2.2') >= 0 :
    Crabpydir=commands.getoutput('which crab')
    Topdir=string.replace(Crabpydir,'/python/crab','')
    extradir=Topdir+'/DLSAPI/extra'
    if sys.path.count(extradir) <= 0:
        if os.path.exists(extradir):
            sys.path.insert(0, extradir)

# #######################################
class DBSError(exceptions.Exception):
    def __init__(self, errorName, errorMessage):
        args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
        exceptions.Exception.__init__(self, args)
        pass
    
    def getErrorMessage(self):
        """ Return error message """
        return "%s" % (self.args)

# #######################################
class DBSInvalidDataTierError(exceptions.Exception):
    def __init__(self, errorName, errorMessage):
        args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
        exceptions.Exception.__init__(self, args)
        pass
    
    def getErrorMessage(self):
        """ Return error message """
        return "%s" % (self.args)

# #######################################
class DBSInfoError:
    def __init__(self, url):
        print '\nERROR accessing DBS url : '+url+'\n'
        pass

##################################################################################
# Class to extract info from DBS 
###############################################################################

class DBSInfo:
    def __init__(self, dbs_url, dbs_instance):
        """
        Construct api object.
        """
        ## cgi service API
        args = {}
        args['instance']=dbs_instance

        common.logger.debug(3,"Accessing DBS at: "+dbs_url+" "+dbs_instance)

        self.api = dbsCgiApi.DbsCgiApi(dbs_url, args)
        ## set log level
        # self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_INFO_)
        #self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_QUIET_)

    def getMatchingDatasets (self, datasetPath):
        """ Query DBS to get provenance """
        try:
            list = self.api.listProcessedDatasets("%s" %datasetPath)
        except dbsApi.InvalidDataTier, ex:
            raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
        except dbsApi.DbsApiException, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        except dbsCgiApi.DbsCgiToolError , ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        except dbsCgiApi.DbsCgiBadResponse , ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())

        return list


    def getDatasetProvenance(self, path, dataTiers):
        """ Query DBS to get provenance """
        try:
            datasetParentList = self.api.getDatasetProvenance(path,dataTiers)
        except dbsApi.InvalidDataTier, ex:
            raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
        except dbsApi.DbsApiException, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        return datasetParentList                                                                                                            

    def getEventsPerBlock(self, path):
        """ Query DBS to get event collections """
        # count events per block
        nevtsbyblock = {}
        try:
            contents = self.api.getDatasetContents(path)
        except dbsApi.DbsApiException, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        except dbsCgiApi.DbsCgiBadResponse, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        for fileBlock in contents:
            ## get the event collections for each block
            nevts = 0
            eventCollectionList = fileBlock.get('eventCollectionList')
            for evc in eventCollectionList:
                nevts = nevts + evc.get('numberOfEvents')

            common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(nevts,fileBlock.get('blockName')))
            nevtsbyblock[fileBlock.get('blockName')]=nevts

        # returning a map of fileblock-nevts  will be enough for now
        # TODO: in future the EvC collections grouped by fileblock should be returned
        return nevtsbyblock

    def getEventsPerFile(self, path):
        """ Query DBS to get a dictionary of files:(events/file) """
        numEventsByFile = {}
        try:
            contents = self.api.getDatasetContents(path)
        except dbsApi.DbsApiException, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        for fileBlock in contents:
            numEvents = 0
            eventCollectionList = fileBlock.get('eventCollectionList')
            for evc in eventCollectionList:
                numEvents = evc.get('numberOfEvents')
                fileList = evc.get('fileList')
                # As of 2006-08-10, event collections contain only one file
                # => fileList contains only one dictionary
                if len(fileList)>1:
                    msg = "Event collection contains more than one file!  Exiting.\n"
                    msg = msg + "CRAB and DBS must be upgraded to handle event collections with multiple files.\n"
                    raise CrabException(msg)
                fileDict = fileList[0]
                fileName = fileDict.get('logicalFileName')
                numEventsByFile[fileName] = numEvents
        return numEventsByFile

    def getDatasetFileBlocks(self, path):
        """ Query DBS to get files/fileblocks """
        try:
            FilesbyBlock={}
            try:
                allBlocks = self.api.getDatasetFileBlocks(path)
            except dbsCgiApi.DbsCgiBadResponse, ex:
                raise DBSError(ex.getClassName(), ex.getErrorMessage())
            for fileBlock in allBlocks:
                blockname=fileBlock.get('blockName')
                filesinblock=[]
                for files in fileBlock.get('fileList'):
                    #print "  block %s has file %s"%(blockname,files.getLogicalFileName())
                    filesinblock.append(files.get('logicalFileName'))
                FilesbyBlock[blockname]=filesinblock
        except dbsApi.DbsApiException, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())

        return FilesbyBlock
Revision:	1.11
Committed:	Wed Sep 20 17:29:52 2006 UTC (18 years, 7 months ago) by slacapra
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_1_5_0_pre3, CRAB_1_5_0_pre2, CRAB_1_4_2, CRAB_1_5_0_pre1, CRAB_1_4_1, CRAB_1_4_1_pre2, CRAB_1_4_1_pre1, CRAB_1_4_0, CRAB_1_4_0_pre4, CRAB_1_4_0_pre3, CRAB_1_4_0_pre2, CRAB_1_4_0_pre1, CRAB_1_3_0, CRAB_1_3_0_pre6, CRAB_1_3_0_pre5, CRAB_1_3_0_pre4, CRAB_1_3_0_pre3
Branch point for:	branch_1_4_1
Changes since 1.10:	+78 -45 lines
Log Message:	BOSS4 + sub-file splitting + taskDB
#	User	Rev	Content
1	afanfani	1.1	#!/usr/bin/env python
2	afanfani	1.2	import sys, os, string, re, commands
3			import exceptions
4	afanfani	1.1	import common
5			from crab_exceptions import *
6			try:
7	slacapra	1.4	import dbsCgiApi
8			import dbsApi
9	afanfani	1.1	except:
10	slacapra	1.4	try:
11			Crabpydir=commands.getoutput('which crab')
12			Topdir=string.replace(Crabpydir,'/python/crab','')
13			sys.path.append(Topdir+'/DBSAPI')
14			import dbsCgiApi
15			import dbsApi
16			except:
17			msg="ERROR no DBS API available"
18			raise CrabException(msg)
19	slacapra	1.11
20	afanfani	1.7	## for python 2.2 add the pyexpat.so to PYTHONPATH
21			pythonV=sys.version.split(' ')[0]
22			if pythonV.find('2.2') >= 0 :
23	slacapra	1.8	Crabpydir=commands.getoutput('which crab')
24			Topdir=string.replace(Crabpydir,'/python/crab','')
25			extradir=Topdir+'/DLSAPI/extra'
26			if sys.path.count(extradir) <= 0:
27			if os.path.exists(extradir):
28			sys.path.insert(0, extradir)
29	afanfani	1.7
30	afanfani	1.2	# #######################################
31			class DBSError(exceptions.Exception):
32	slacapra	1.4	def __init__(self, errorName, errorMessage):
33			args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
34			exceptions.Exception.__init__(self, args)
35			pass
36	slacapra	1.11
37	slacapra	1.4	def getErrorMessage(self):
38	slacapra	1.11	""" Return error message """
39	slacapra	1.4	return "%s" % (self.args)
40	afanfani	1.2
41			# #######################################
42			class DBSInvalidDataTierError(exceptions.Exception):
43	slacapra	1.4	def __init__(self, errorName, errorMessage):
44			args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
45			exceptions.Exception.__init__(self, args)
46			pass
47	slacapra	1.11
48	slacapra	1.4	def getErrorMessage(self):
49	slacapra	1.11	""" Return error message """
50	slacapra	1.4	return "%s" % (self.args)
51	afanfani	1.1
52	slacapra	1.11	# #######################################
53	afanfani	1.1	class DBSInfoError:
54	slacapra	1.4	def __init__(self, url):
55			print '\nERROR accessing DBS url : '+url+'\n'
56			pass
57	afanfani	1.1
58			##################################################################################
59			# Class to extract info from DBS
60			###############################################################################
61
62			class DBSInfo:
63	slacapra	1.11	def __init__(self, dbs_url, dbs_instance):
64			"""
65			Construct api object.
66			"""
67			## cgi service API
68			args = {}
69			args['instance']=dbs_instance
70
71			common.logger.debug(3,"Accessing DBS at: "+dbs_url+" "+dbs_instance)
72
73			self.api = dbsCgiApi.DbsCgiApi(dbs_url, args)
74			## set log level
75	slacapra	1.4	# self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_INFO_)
76	slacapra	1.11	#self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_QUIET_)
77	afanfani	1.1
78	slacapra	1.11	def getMatchingDatasets (self, datasetPath):
79	slacapra	1.4	""" Query DBS to get provenance """
80			try:
81	slacapra	1.11	list = self.api.listProcessedDatasets("%s" %datasetPath)
82	slacapra	1.4	except dbsApi.InvalidDataTier, ex:
83			raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
84			except dbsApi.DbsApiException, ex:
85			raise DBSError(ex.getClassName(),ex.getErrorMessage())
86	afanfani	1.5	except dbsCgiApi.DbsCgiToolError , ex:
87			raise DBSError(ex.getClassName(),ex.getErrorMessage())
88	slacapra	1.11	except dbsCgiApi.DbsCgiBadResponse , ex:
89	afanfani	1.6	raise DBSError(ex.getClassName(),ex.getErrorMessage())
90	afanfani	1.5
91	slacapra	1.4	return list
92	afanfani	1.3
93	slacapra	1.11
94	slacapra	1.4	def getDatasetProvenance(self, path, dataTiers):
95	slacapra	1.11	""" Query DBS to get provenance """
96	slacapra	1.4	try:
97			datasetParentList = self.api.getDatasetProvenance(path,dataTiers)
98			except dbsApi.InvalidDataTier, ex:
99	slacapra	1.11	raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
100	slacapra	1.4	except dbsApi.DbsApiException, ex:
101			raise DBSError(ex.getClassName(),ex.getErrorMessage())
102			return datasetParentList
103	afanfani	1.1
104	mkirn	1.10	def getEventsPerBlock(self, path):
105	slacapra	1.11	""" Query DBS to get event collections """
106			# count events per block
107			nevtsbyblock = {}
108	slacapra	1.4	try:
109	slacapra	1.11	contents = self.api.getDatasetContents(path)
110	slacapra	1.4	except dbsApi.DbsApiException, ex:
111			raise DBSError(ex.getClassName(),ex.getErrorMessage())
112	slacapra	1.11	except dbsCgiApi.DbsCgiBadResponse, ex:
113	afanfani	1.6	raise DBSError(ex.getClassName(),ex.getErrorMessage())
114	slacapra	1.11	for fileBlock in contents:
115	afanfani	1.1	## get the event collections for each block
116	slacapra	1.11	nevts = 0
117	afanfani	1.5	eventCollectionList = fileBlock.get('eventCollectionList')
118	slacapra	1.11	for evc in eventCollectionList:
119			nevts = nevts + evc.get('numberOfEvents')
120
121	afanfani	1.5	common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(nevts,fileBlock.get('blockName')))
122			nevtsbyblock[fileBlock.get('blockName')]=nevts
123	afanfani	1.1
124	slacapra	1.4	# returning a map of fileblock-nevts will be enough for now
125			# TODO: in future the EvC collections grouped by fileblock should be returned
126			return nevtsbyblock
127	afanfani	1.1
128	slacapra	1.11	def getEventsPerFile(self, path):
129			""" Query DBS to get a dictionary of files:(events/file) """
130			numEventsByFile = {}
131			try:
132			contents = self.api.getDatasetContents(path)
133			except dbsApi.DbsApiException, ex:
134			raise DBSError(ex.getClassName(),ex.getErrorMessage())
135			for fileBlock in contents:
136			numEvents = 0
137			eventCollectionList = fileBlock.get('eventCollectionList')
138			for evc in eventCollectionList:
139			numEvents = evc.get('numberOfEvents')
140			fileList = evc.get('fileList')
141			# As of 2006-08-10, event collections contain only one file
142			# => fileList contains only one dictionary
143			if len(fileList)>1:
144			msg = "Event collection contains more than one file! Exiting.\n"
145			msg = msg + "CRAB and DBS must be upgraded to handle event collections with multiple files.\n"
146			raise CrabException(msg)
147			fileDict = fileList[0]
148			fileName = fileDict.get('logicalFileName')
149			numEventsByFile[fileName] = numEvents
150			return numEventsByFile
151
152			def getDatasetFileBlocks(self, path):
153			""" Query DBS to get files/fileblocks """
154			try:
155			FilesbyBlock={}
156			try:
157			allBlocks = self.api.getDatasetFileBlocks(path)
158			except dbsCgiApi.DbsCgiBadResponse, ex:
159			raise DBSError(ex.getClassName(), ex.getErrorMessage())
160			for fileBlock in allBlocks:
161			blockname=fileBlock.get('blockName')
162			filesinblock=[]
163			for files in fileBlock.get('fileList'):
164			#print " block %s has file %s"%(blockname,files.getLogicalFileName())
165			filesinblock.append(files.get('logicalFileName'))
166			FilesbyBlock[blockname]=filesinblock
167			except dbsApi.DbsApiException, ex:
168			raise DBSError(ex.getClassName(),ex.getErrorMessage())
169
170			return FilesbyBlock