CRAB/python/DBSInfo.py

#!/usr/bin/env python
import sys, os, string, commands
import exceptions
import common
from crab_exceptions import *
import DBSAPIOLD.dbsCgiApi
import DBSAPIOLD.dbsApi
                                                                                              
## for python 2.2 add the pyexpat.so to PYTHONPATH
pythonV=sys.version.split(' ')[0]
if pythonV.find('2.2') >= 0 :
    Crabpydir=commands.getoutput('which crab')
    Topdir=string.replace(Crabpydir,'/python/crab','')
    extradir=Topdir+'/DLSAPI/extra'
    if sys.path.count(extradir) <= 0:
        if os.path.exists(extradir):
            sys.path.insert(0, extradir)

# #######################################
class DBSError(exceptions.Exception):
    def __init__(self, errorName, errorMessage):
        self.args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
        exceptions.Exception.__init__(self, self.args)
        pass
    
    def getErrorMessage(self):
        """ Return error message """
        return "%s" % (self.args)

# #######################################
class DBSInvalidDataTierError(exceptions.Exception):
    def __init__(self, errorName, errorMessage):
        self.args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
        exceptions.Exception.__init__(self, self.args)
        pass
    
    def getErrorMessage(self):
        """ Return error message """
        return "%s" % (self.args)

# #######################################
class DBSInfoError:
    def __init__(self, url):
        print '\nERROR accessing DBS url : '+url+'\n'
        pass

##################################################################################
# Class to extract info from DBS 
###############################################################################

class DBSInfo:
    def __init__(self, dbs_url, dbs_instance):
        """
        Construct api object.
        """
        ## cgi service API
        args = {}
        args['instance']=dbs_instance

        common.logger.debug(3,"Accessing DBS at: "+dbs_url+" "+dbs_instance)

        self.api = DBSAPIOLD.dbsCgiApi.DbsCgiApi(dbs_url, args)
        ## set log level
        # self.api.setLogLevel(DBSAPIOLD.dbsApi.DBS_LOG_LEVEL_INFO_)
        #self.api.setLogLevel(DBSAPIOLD.dbsApi.DBS_LOG_LEVEL_QUIET_)

    def getMatchingDatasets (self, datasetPath):
        """ Query DBS to get provenance """
        try:
            result = self.api.listProcessedDatasets("%s" %datasetPath)
        except DBSAPIOLD.dbsApi.InvalidDataTier, ex:
            raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
        except DBSAPIOLD.dbsApi.DbsApiException, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        except DBSAPIOLD.dbsCgiApi.DbsCgiToolError , ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        except DBSAPIOLD.dbsCgiApi.DbsCgiBadResponse , ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())

        return result


    def getDatasetProvenance(self, path, dataTiers):
        """ Query DBS to get provenance """
        try:
            datasetParentList = self.api.getDatasetProvenance(path,dataTiers)
        except DBSAPIOLD.dbsApi.InvalidDataTier, ex:
            raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
        except DBSAPIOLD.dbsApi.DbsApiException, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        return datasetParentList                                                                                                            

    def getEventsPerBlock(self, path):
        """ Query DBS to get event collections """
        # count events per block
        nevtsbyblock = {}
        try:
            contents = self.api.getDatasetContents(path)
        except DBSAPIOLD.dbsApi.DbsApiException, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        except DBSAPIOLD.dbsCgiApi.DbsCgiBadResponse, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        for fileBlock in contents:
            ## get the event collections for each block
            nevts = 0
            eventCollectionList = fileBlock.get('eventCollectionList')
            for evc in eventCollectionList:
                nevts = nevts + evc.get('numberOfEvents')

            common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(nevts,fileBlock.get('blockName')))
            nevtsbyblock[fileBlock.get('blockName')]=nevts

        # returning a map of fileblock-nevts  will be enough for now
        # TODO: in future the EvC collections grouped by fileblock should be returned
        return nevtsbyblock

    def getEventsPerFile(self, path):
        """ Query DBS to get a dictionary of files:(events/file) """
        numEventsByFile = {}
        try:
            contents = self.api.getDatasetContents(path)
        except DBSAPIOLD.dbsApi.DbsApiException, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())
        for fileBlock in contents:
            numEvents = 0
            eventCollectionList = fileBlock.get('eventCollectionList')
            for evc in eventCollectionList:
                numEvents = evc.get('numberOfEvents')
                fileList = evc.get('fileList')
                # As of 2006-08-10, event collections contain only one file
                # => fileList contains only one dictionary
                if len(fileList)>1:
                    msg = "Event collection contains more than one file!  Exiting.\n"
                    msg = msg + "CRAB and DBS must be upgraded to handle event collections with multiple files.\n"
                    raise CrabException(msg)
                fileDict = fileList[0]
                fileName = fileDict.get('logicalFileName')
                numEventsByFile[fileName] = numEvents
        return numEventsByFile

    def getDatasetFileBlocks(self, path):
        """ Query DBS to get files/fileblocks """
        try:
            FilesbyBlock={}
            try:
                allBlocks = self.api.getDatasetFileBlocks(path)
            except DBSAPIOLD.dbsCgiApi.DbsCgiBadResponse, ex:
                raise DBSError(ex.getClassName(), ex.getErrorMessage())
            for fileBlock in allBlocks:
                blockname=fileBlock.get('blockName')
                filesinblock=[]
                for files in fileBlock.get('fileList'):
                    #print "  block %s has file %s"%(blockname,files.getLogicalFileName())
                    filesinblock.append(files.get('logicalFileName'))
                FilesbyBlock[blockname]=filesinblock
        except DBSAPIOLD.dbsApi.DbsApiException, ex:
            raise DBSError(ex.getClassName(),ex.getErrorMessage())

        return FilesbyBlock
Revision:	1.14
Committed:	Wed Feb 7 00:15:38 2007 UTC (18 years, 2 months ago) by gutsche
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_2_4_0_test, CRAB_2_3_1, CRAB_2_3_1_pre6, CRAB_2_3_1_pre5, CRAB_2_3_1_pre4, CRAB_2_3_1_pre3, CRAB_2_3_1_pre2, CRAB_2_3_1_pre1, CRAB_2_3_0, CRAB_2_3_0_pre6, CRAB_2_3_0_pre1, CRAB_2_2_2_pre5, CRAB_2_2_2_pre4, CRAB_2_2_2_pre3, CRAB_2_2_2_pre2, CRAB_2_2_2_pre1, CRAB_2_2_1, CRAB_2_2_1_pre6, CRAB_2_2_1_pre5, CRAB_2_2_1_pre4, PRODCOMMON_0_10_7_testCS2, CRAB_2_2_1_pre3, CRAB_2_2_1_pre2, CRAB_2_2_1_pre1, CRAB_2_2_0, CRAB_2_2_0_pre21, CRAB_2_2_0_pre19, CRAB_2_2_0_pre18, CRAB_2_2_0_pre17, CRAB_2_2_0_pre16, CRAB_2_2_0_pre15, CRAB_2_2_0_pre13, CRAB_2_2_0_pre12, CRAB_2_2_0_pre11, CRAB_2_2_0_pre10, bp_osg_bdii, CRAB_2_2_0_pre9, CRAB_2_2_0_pre8, CRAB_2_2_0_pre7, CRAB_2_1_2, CRAB_2_2_0_pre5, CRAB_2_1_2_pre2, CRAB_2_1_2_pre1, CRAB_2_2_0_pre4, CRAB_2_2_0_pre2, CRAB_2_1_1, CRAB_2_1_1_pre3, CRAB_2_2_0_pre1, CRAB_2_1_1_pre1, CRAB_2_1_0, CRAB_2_1_0_pre6, CRAB_2_1_0_pre5, CRAB_2_1_0_pre4, CRAB_2_1_0_pre3, CRAB_2_1_0_pre2, CRAB_2_1_0_pre1, CRAB_2_0_4, CRAB_2_0_4_pre2, CRAB_2_0_4_pre1, CRAB_2_0_3, CRAB_2_0_3_pre1, CRAB_2_0_2, CRAB_2_0_2_pre6, CRAB_2_0_2_pre5, CRAB_2_0_2_pre4, CRAB_2_0_2_pre3, CRAB_1_5_4_SLC3, CRAB_1_5_4_SLC3_pre4, CRAB_2_0_2_pre2, CRAB_2_0_2_pre1, CRAB_1_5_4_SLC3_pre3, CRAB_2_0_1, CRAB_1_5_4_SLC3_pre2, CRAB_2_0_1_pre1, CRAB_1_5_4_SLC3_pre1, CRAB_2_0_0, CRAB_2_0_0_pre10, CRAB_2_0_0_pre9, CRAB_1_5_4, CRAB_1_5_4_pre2, CRAB_1_5_4_pre1, CRAB_2_0_0_pre7, CRAB_2_0_0_pre6, CRAB_1_5_3, CRAB_1_5_3_pre5, CRAB_1_5_3_pre4, CRAB_2_0_0_pre5, CRAB_1_5_3_pre3, configure, CRAB_2_0_0_pre4, CRAB_1_5_3_pre2, CRAB_1_5_3_pre1, CRAB_2_0_0_pre3, CRAB_1_5_2, CRAB_2_0_0_pre2, CRAB_2_0_0_pre1, CRAB_1_5_1, CRAB_1_5_1_pre4, CRAB_1_5_1_pre3, CRAB_1_5_1_pre2, CRAB_1_5_1_pre1, CRAB_1_5_0, CRAB_1_5_0_pre9, CRAB_1_5_0_pre8, CRAB_1_5_0_pre7
Branch point for:	CRAB_2_3_0_br, osg_bdii, CRAB_2_1_2_br, CRAB_2_1_1_pre2, CRAB_1_5_4_SLC3_pre4_br, CRAB_1_5_4_SLC3_start
Changes since 1.13:	+16 -16 lines
Log Message:	transition to new DBS api which follows python namespaces
#	User	Rev	Content
1	afanfani	1.1	#!/usr/bin/env python
2	slacapra	1.12	import sys, os, string, commands
3	afanfani	1.2	import exceptions
4	afanfani	1.1	import common
5			from crab_exceptions import *
6	gutsche	1.14	import DBSAPIOLD.dbsCgiApi
7			import DBSAPIOLD.dbsApi
8	slacapra	1.11
9	afanfani	1.7	## for python 2.2 add the pyexpat.so to PYTHONPATH
10			pythonV=sys.version.split(' ')[0]
11			if pythonV.find('2.2') >= 0 :
12	slacapra	1.8	Crabpydir=commands.getoutput('which crab')
13			Topdir=string.replace(Crabpydir,'/python/crab','')
14			extradir=Topdir+'/DLSAPI/extra'
15			if sys.path.count(extradir) <= 0:
16			if os.path.exists(extradir):
17			sys.path.insert(0, extradir)
18	afanfani	1.7
19	afanfani	1.2	# #######################################
20			class DBSError(exceptions.Exception):
21	slacapra	1.4	def __init__(self, errorName, errorMessage):
22	gutsche	1.13	self.args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
23	slacapra	1.12	exceptions.Exception.__init__(self, self.args)
24	slacapra	1.4	pass
25	slacapra	1.11
26	slacapra	1.4	def getErrorMessage(self):
27	slacapra	1.11	""" Return error message """
28	slacapra	1.4	return "%s" % (self.args)
29	afanfani	1.2
30			# #######################################
31			class DBSInvalidDataTierError(exceptions.Exception):
32	slacapra	1.4	def __init__(self, errorName, errorMessage):
33	gutsche	1.13	self.args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
34	slacapra	1.12	exceptions.Exception.__init__(self, self.args)
35	slacapra	1.4	pass
36	slacapra	1.11
37	slacapra	1.4	def getErrorMessage(self):
38	slacapra	1.11	""" Return error message """
39	slacapra	1.4	return "%s" % (self.args)
40	afanfani	1.1
41	slacapra	1.11	# #######################################
42	afanfani	1.1	class DBSInfoError:
43	slacapra	1.4	def __init__(self, url):
44			print '\nERROR accessing DBS url : '+url+'\n'
45			pass
46	afanfani	1.1
47			##################################################################################
48			# Class to extract info from DBS
49			###############################################################################
50
51			class DBSInfo:
52	slacapra	1.11	def __init__(self, dbs_url, dbs_instance):
53			"""
54			Construct api object.
55			"""
56			## cgi service API
57			args = {}
58			args['instance']=dbs_instance
59
60			common.logger.debug(3,"Accessing DBS at: "+dbs_url+" "+dbs_instance)
61
62	gutsche	1.14	self.api = DBSAPIOLD.dbsCgiApi.DbsCgiApi(dbs_url, args)
63	slacapra	1.11	## set log level
64	gutsche	1.14	# self.api.setLogLevel(DBSAPIOLD.dbsApi.DBS_LOG_LEVEL_INFO_)
65			#self.api.setLogLevel(DBSAPIOLD.dbsApi.DBS_LOG_LEVEL_QUIET_)
66	afanfani	1.1
67	slacapra	1.11	def getMatchingDatasets (self, datasetPath):
68	slacapra	1.4	""" Query DBS to get provenance """
69			try:
70	slacapra	1.12	result = self.api.listProcessedDatasets("%s" %datasetPath)
71	gutsche	1.14	except DBSAPIOLD.dbsApi.InvalidDataTier, ex:
72	slacapra	1.4	raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
73	gutsche	1.14	except DBSAPIOLD.dbsApi.DbsApiException, ex:
74	slacapra	1.4	raise DBSError(ex.getClassName(),ex.getErrorMessage())
75	gutsche	1.14	except DBSAPIOLD.dbsCgiApi.DbsCgiToolError , ex:
76	afanfani	1.5	raise DBSError(ex.getClassName(),ex.getErrorMessage())
77	gutsche	1.14	except DBSAPIOLD.dbsCgiApi.DbsCgiBadResponse , ex:
78	afanfani	1.6	raise DBSError(ex.getClassName(),ex.getErrorMessage())
79	afanfani	1.5
80	slacapra	1.12	return result
81	afanfani	1.3
82	slacapra	1.11
83	slacapra	1.4	def getDatasetProvenance(self, path, dataTiers):
84	slacapra	1.11	""" Query DBS to get provenance """
85	slacapra	1.4	try:
86			datasetParentList = self.api.getDatasetProvenance(path,dataTiers)
87	gutsche	1.14	except DBSAPIOLD.dbsApi.InvalidDataTier, ex:
88	slacapra	1.11	raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
89	gutsche	1.14	except DBSAPIOLD.dbsApi.DbsApiException, ex:
90	slacapra	1.4	raise DBSError(ex.getClassName(),ex.getErrorMessage())
91			return datasetParentList
92	afanfani	1.1
93	mkirn	1.10	def getEventsPerBlock(self, path):
94	slacapra	1.11	""" Query DBS to get event collections """
95			# count events per block
96			nevtsbyblock = {}
97	slacapra	1.4	try:
98	slacapra	1.11	contents = self.api.getDatasetContents(path)
99	gutsche	1.14	except DBSAPIOLD.dbsApi.DbsApiException, ex:
100	slacapra	1.4	raise DBSError(ex.getClassName(),ex.getErrorMessage())
101	gutsche	1.14	except DBSAPIOLD.dbsCgiApi.DbsCgiBadResponse, ex:
102	afanfani	1.6	raise DBSError(ex.getClassName(),ex.getErrorMessage())
103	slacapra	1.11	for fileBlock in contents:
104	afanfani	1.1	## get the event collections for each block
105	slacapra	1.11	nevts = 0
106	afanfani	1.5	eventCollectionList = fileBlock.get('eventCollectionList')
107	slacapra	1.11	for evc in eventCollectionList:
108			nevts = nevts + evc.get('numberOfEvents')
109
110	afanfani	1.5	common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(nevts,fileBlock.get('blockName')))
111			nevtsbyblock[fileBlock.get('blockName')]=nevts
112	afanfani	1.1
113	slacapra	1.4	# returning a map of fileblock-nevts will be enough for now
114			# TODO: in future the EvC collections grouped by fileblock should be returned
115			return nevtsbyblock
116	afanfani	1.1
117	slacapra	1.11	def getEventsPerFile(self, path):
118			""" Query DBS to get a dictionary of files:(events/file) """
119			numEventsByFile = {}
120			try:
121			contents = self.api.getDatasetContents(path)
122	gutsche	1.14	except DBSAPIOLD.dbsApi.DbsApiException, ex:
123	slacapra	1.11	raise DBSError(ex.getClassName(),ex.getErrorMessage())
124			for fileBlock in contents:
125			numEvents = 0
126			eventCollectionList = fileBlock.get('eventCollectionList')
127			for evc in eventCollectionList:
128			numEvents = evc.get('numberOfEvents')
129			fileList = evc.get('fileList')
130			# As of 2006-08-10, event collections contain only one file
131			# => fileList contains only one dictionary
132			if len(fileList)>1:
133			msg = "Event collection contains more than one file! Exiting.\n"
134			msg = msg + "CRAB and DBS must be upgraded to handle event collections with multiple files.\n"
135			raise CrabException(msg)
136			fileDict = fileList[0]
137			fileName = fileDict.get('logicalFileName')
138			numEventsByFile[fileName] = numEvents
139			return numEventsByFile
140
141			def getDatasetFileBlocks(self, path):
142			""" Query DBS to get files/fileblocks """
143			try:
144			FilesbyBlock={}
145			try:
146			allBlocks = self.api.getDatasetFileBlocks(path)
147	gutsche	1.14	except DBSAPIOLD.dbsCgiApi.DbsCgiBadResponse, ex:
148	slacapra	1.11	raise DBSError(ex.getClassName(), ex.getErrorMessage())
149			for fileBlock in allBlocks:
150			blockname=fileBlock.get('blockName')
151			filesinblock=[]
152			for files in fileBlock.get('fileList'):
153			#print " block %s has file %s"%(blockname,files.getLogicalFileName())
154			filesinblock.append(files.get('logicalFileName'))
155			FilesbyBlock[blockname]=filesinblock
156	gutsche	1.14	except DBSAPIOLD.dbsApi.DbsApiException, ex:
157	slacapra	1.11	raise DBSError(ex.getClassName(),ex.getErrorMessage())
158
159			return FilesbyBlock