CRAB/python/DataDiscovery_DBS2.py

#!/usr/bin/env python
import exceptions
import DBSAPI.dbsApi
from DBSAPI.dbsApiException import * 
import common


# #######################################
class DBSError_DBS2(exceptions.Exception):
    def __init__(self, errorName, errorMessage):
        args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
        exceptions.Exception.__init__(self, args)
        pass
    
    def getErrorMessage(self):
        """ Return error message """
        return "%s" % (self.args)

# #######################################
class DBSInvalidDataTierError_DBS2(exceptions.Exception):
    def __init__(self, errorName, errorMessage):
        args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
        exceptions.Exception.__init__(self, args)
        pass
    
    def getErrorMessage(self):
        """ Return error message """
        return "%s" % (self.args)

# #######################################
class DBSInfoError_DBS2:
    def __init__(self, url):
        print '\nERROR accessing DBS url : '+url+'\n'
        pass

# ####################################
class DataDiscoveryError_DBS2(exceptions.Exception):
    def __init__(self, errorMessage):
        self.args=errorMessage
        exceptions.Exception.__init__(self, self.args)
        pass

    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
class NotExistingDatasetError_DBS2(exceptions.Exception):
    def __init__(self, errorMessage):
        self.args=errorMessage
        exceptions.Exception.__init__(self, self.args)
        pass

    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
class NoDataTierinProvenanceError_DBS2(exceptions.Exception):
    def __init__(self, errorMessage):
        self.args=errorMessage
        exceptions.Exception.__init__(self, self.args)
        pass

    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
# class to find and extact info from published data
class DataDiscovery_DBS2:
    def __init__(self, datasetPath, cfg_params):

        #       Attributes
        self.datasetPath = datasetPath
        self.cfg_params = cfg_params

        self.eventsPerBlock = {}  # DBS output: map fileblocks-events for collection
        self.eventsPerFile = {}   # DBS output: map files-events
        self.blocksinfo = {}      # DBS output: map fileblocks-files 
        self.maxEvents = 0        # DBS output: max events

# ####################################
    def fetchDBSInfo(self):
        """
        Contact DBS
        """

        ## get DBS URL
        try:
            dbs_url=self.cfg_params['CMSSW.dbs_url']
        except KeyError:
            dbs_url="http://cmsdoc.cern.ch/cms/test/aprom/DBS/CGIServer/prodquery"

        ## get DBS URL
        try:
            dbs_version=self.cfg_params['CMSSW.dbs_version']
        except KeyError:
            dbs_version="v00_00_05"

        ## service API
        args = {}
        args['url']     = dbs_url
        args['version'] = dbs_version

        common.logger.debug(3,"Accessing DBS at: "+dbs_url+" with version: "+dbs_version)

        api = DBSAPI.dbsApi.DbsApi(args)
        try:
            files = api.listFiles(self.datasetPath)
        except DbsBadRequest, msg:
            raise DataDiscoveryError_DBS2(msg)
        except DBSError_DBS2, msg:
            raise DataDiscoveryError_DBS2(msg)
        
        # parse files and fill arrays
        for file in files :
            filename = file['LogicalFileName']
            events = file['NumberOfEvents']
            fileblock = file['Block']['Name']

            # number of events per block
            if fileblock in self.eventsPerBlock.keys() :
                self.eventsPerBlock[fileblock] += events
            else :
                self.eventsPerBlock[fileblock] = events

            # number of events per file
            self.eventsPerFile[filename] = events

            # number of events per block
            if fileblock in self.blocksinfo.keys() :
                self.blocksinfo[fileblock].append(filename)
            else :
                self.blocksinfo[fileblock] = [filename]

            # total number of events
            self.maxEvents += events

        if len(self.eventsPerBlock) <= 0:
            raise NotExistingDatasetError_DBS2 (("\nNo data for %s in DBS\nPlease check"
                                            + " dataset path variables in crab.cfg")
                                            % self.datasetPath)


# #################################################
    def getMaxEvents(self):
        """
        max events 
        """
        return self.maxEvents

# #################################################
    def getEventsPerBlock(self):
        """
        list the event collections structure by fileblock 
        """
        return self.eventsPerBlock

# #################################################
    def getEventsPerFile(self):
        """
        list the event collections structure by file 
        """
        return self.eventsPerFile

# #################################################
    def getFiles(self):
        """
        return files grouped by fileblock 
        """
        return self.blocksinfo        

########################################################################
Revision:	1.3
Committed:	Wed Feb 7 16:47:45 2007 UTC (18 years, 2 months ago) by gutsche
Content type:	text/x-python
Branch:	MAIN
Changes since 1.2:	+9 -2 lines
Log Message:	Add dbs_version card for steering the api version of the selected dbs server, default v00_00_05. Improved exception handling
#	User	Rev	Content
1	gutsche	1.1	#!/usr/bin/env python
2			import exceptions
3	gutsche	1.2	import DBSAPI.dbsApi
4			from DBSAPI.dbsApiException import *
5	gutsche	1.1	import common
6
7
8			# #######################################
9			class DBSError_DBS2(exceptions.Exception):
10			def __init__(self, errorName, errorMessage):
11			args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
12			exceptions.Exception.__init__(self, args)
13			pass
14
15			def getErrorMessage(self):
16			""" Return error message """
17			return "%s" % (self.args)
18
19			# #######################################
20			class DBSInvalidDataTierError_DBS2(exceptions.Exception):
21			def __init__(self, errorName, errorMessage):
22			args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
23			exceptions.Exception.__init__(self, args)
24			pass
25
26			def getErrorMessage(self):
27			""" Return error message """
28			return "%s" % (self.args)
29
30			# #######################################
31			class DBSInfoError_DBS2:
32			def __init__(self, url):
33			print '\nERROR accessing DBS url : '+url+'\n'
34			pass
35
36			# ####################################
37			class DataDiscoveryError_DBS2(exceptions.Exception):
38			def __init__(self, errorMessage):
39			self.args=errorMessage
40			exceptions.Exception.__init__(self, self.args)
41			pass
42
43			def getErrorMessage(self):
44			""" Return exception error """
45			return "%s" % (self.args)
46
47			# ####################################
48			class NotExistingDatasetError_DBS2(exceptions.Exception):
49			def __init__(self, errorMessage):
50			self.args=errorMessage
51			exceptions.Exception.__init__(self, self.args)
52			pass
53
54			def getErrorMessage(self):
55			""" Return exception error """
56			return "%s" % (self.args)
57
58			# ####################################
59			class NoDataTierinProvenanceError_DBS2(exceptions.Exception):
60			def __init__(self, errorMessage):
61			self.args=errorMessage
62			exceptions.Exception.__init__(self, self.args)
63			pass
64
65			def getErrorMessage(self):
66			""" Return exception error """
67			return "%s" % (self.args)
68
69			# ####################################
70			# class to find and extact info from published data
71			class DataDiscovery_DBS2:
72			def __init__(self, datasetPath, cfg_params):
73
74			# Attributes
75			self.datasetPath = datasetPath
76			self.cfg_params = cfg_params
77
78			self.eventsPerBlock = {} # DBS output: map fileblocks-events for collection
79			self.eventsPerFile = {} # DBS output: map files-events
80			self.blocksinfo = {} # DBS output: map fileblocks-files
81			self.maxEvents = 0 # DBS output: max events
82
83			# ####################################
84			def fetchDBSInfo(self):
85			"""
86			Contact DBS
87			"""
88
89			## get DBS URL
90			try:
91			dbs_url=self.cfg_params['CMSSW.dbs_url']
92			except KeyError:
93			dbs_url="http://cmsdoc.cern.ch/cms/test/aprom/DBS/CGIServer/prodquery"
94
95	gutsche	1.3	## get DBS URL
96			try:
97			dbs_version=self.cfg_params['CMSSW.dbs_version']
98			except KeyError:
99			dbs_version="v00_00_05"
100
101	gutsche	1.1	## service API
102			args = {}
103	gutsche	1.3	args['url'] = dbs_url
104			args['version'] = dbs_version
105	gutsche	1.1
106	gutsche	1.3	common.logger.debug(3,"Accessing DBS at: "+dbs_url+" with version: "+dbs_version)
107	gutsche	1.1
108	gutsche	1.2	api = DBSAPI.dbsApi.DbsApi(args)
109	gutsche	1.1	try:
110			files = api.listFiles(self.datasetPath)
111			except DbsBadRequest, msg:
112	gutsche	1.2	raise DataDiscoveryError_DBS2(msg)
113	gutsche	1.1	except DBSError_DBS2, msg:
114	gutsche	1.2	raise DataDiscoveryError_DBS2(msg)
115	gutsche	1.1
116			# parse files and fill arrays
117			for file in files :
118			filename = file['LogicalFileName']
119			events = file['NumberOfEvents']
120			fileblock = file['Block']['Name']
121
122			# number of events per block
123			if fileblock in self.eventsPerBlock.keys() :
124			self.eventsPerBlock[fileblock] += events
125			else :
126			self.eventsPerBlock[fileblock] = events
127
128			# number of events per file
129			self.eventsPerFile[filename] = events
130
131			# number of events per block
132			if fileblock in self.blocksinfo.keys() :
133			self.blocksinfo[fileblock].append(filename)
134			else :
135			self.blocksinfo[fileblock] = [filename]
136
137			# total number of events
138			self.maxEvents += events
139
140			if len(self.eventsPerBlock) <= 0:
141	gutsche	1.2	raise NotExistingDatasetError_DBS2 (("\nNo data for %s in DBS\nPlease check"
142	gutsche	1.1	+ " dataset path variables in crab.cfg")
143			% self.datasetPath)
144
145
146			# #################################################
147			def getMaxEvents(self):
148			"""
149			max events
150			"""
151			return self.maxEvents
152
153			# #################################################
154			def getEventsPerBlock(self):
155			"""
156			list the event collections structure by fileblock
157			"""
158			return self.eventsPerBlock
159
160			# #################################################
161			def getEventsPerFile(self):
162			"""
163			list the event collections structure by file
164			"""
165			return self.eventsPerFile
166
167			# #################################################
168			def getFiles(self):
169			"""
170			return files grouped by fileblock
171			"""
172			return self.blocksinfo
173
174			########################################################################