CRAB/python/DataDiscovery_DBS2.py

#!/usr/bin/env python
import exceptions
import DBS2API.dbsApi
from DBS2API.dbsApiException import * 
import common


# #######################################
class DBSError_DBS2(exceptions.Exception):
    def __init__(self, errorName, errorMessage):
        args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
        exceptions.Exception.__init__(self, args)
        pass
    
    def getErrorMessage(self):
        """ Return error message """
        return "%s" % (self.args)

# #######################################
class DBSInvalidDataTierError_DBS2(exceptions.Exception):
    def __init__(self, errorName, errorMessage):
        args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
        exceptions.Exception.__init__(self, args)
        pass
    
    def getErrorMessage(self):
        """ Return error message """
        return "%s" % (self.args)

# #######################################
class DBSInfoError_DBS2:
    def __init__(self, url):
        print '\nERROR accessing DBS url : '+url+'\n'
        pass

# ####################################
class DataDiscoveryError_DBS2(exceptions.Exception):
    def __init__(self, errorMessage):
        self.args=errorMessage
        exceptions.Exception.__init__(self, self.args)
        pass

    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
class NotExistingDatasetError_DBS2(exceptions.Exception):
    def __init__(self, errorMessage):
        self.args=errorMessage
        exceptions.Exception.__init__(self, self.args)
        pass

    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
class NoDataTierinProvenanceError_DBS2(exceptions.Exception):
    def __init__(self, errorMessage):
        self.args=errorMessage
        exceptions.Exception.__init__(self, self.args)
        pass

    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
# class to find and extact info from published data
class DataDiscovery_DBS2:
    def __init__(self, datasetPath, cfg_params):

        #       Attributes
        self.datasetPath = datasetPath
        self.cfg_params = cfg_params

        self.eventsPerBlock = {}  # DBS output: map fileblocks-events for collection
        self.eventsPerFile = {}   # DBS output: map files-events
        self.blocksinfo = {}      # DBS output: map fileblocks-files 
        self.maxEvents = 0        # DBS output: max events

# ####################################
    def fetchDBSInfo(self):
        """
        Contact DBS
        """

        ## get DBS URL
        try:
            dbs_url=self.cfg_params['CMSSW.dbs_url']
        except KeyError:
            dbs_url="http://cmsdoc.cern.ch/cms/test/aprom/DBS/CGIServer/prodquery"

        ## service API
        args = {}
        args['url']=dbs_url

        common.logger.debug(3,"Accessing DBS at: "+dbs_url)

        api = DBS2API.dbsApi.DbsApi(args)
        try:
            files = api.listFiles(self.datasetPath)
        except DbsBadRequest, msg:
            raise DataDiscoveryError(msg)
        except DBSError_DBS2, msg:
            raise DataDiscoveryError(msg)
        
        # parse files and fill arrays
        for file in files :
            filename = file['LogicalFileName']
            events = file['NumberOfEvents']
            fileblock = file['Block']['Name']

            # number of events per block
            if fileblock in self.eventsPerBlock.keys() :
                self.eventsPerBlock[fileblock] += events
            else :
                self.eventsPerBlock[fileblock] = events

            # number of events per file
            self.eventsPerFile[filename] = events

            # number of events per block
            if fileblock in self.blocksinfo.keys() :
                self.blocksinfo[fileblock].append(filename)
            else :
                self.blocksinfo[fileblock] = [filename]

            # total number of events
            self.maxEvents += events

        if len(self.eventsPerBlock) <= 0:
            raise NotExistingDatasetError (("\nNo data for %s in DBS\nPlease check"
                                            + " dataset path variables in crab.cfg")
                                            % self.datasetPath)


# #################################################
    def getMaxEvents(self):
        """
        max events 
        """
        return self.maxEvents

# #################################################
    def getEventsPerBlock(self):
        """
        list the event collections structure by fileblock 
        """
        return self.eventsPerBlock

# #################################################
    def getEventsPerFile(self):
        """
        list the event collections structure by file 
        """
        return self.eventsPerFile

# #################################################
    def getFiles(self):
        """
        return files grouped by fileblock 
        """
        return self.blocksinfo        

########################################################################
Revision:	1.1
Committed:	Thu Feb 1 16:09:20 2007 UTC (18 years, 3 months ago) by gutsche
Content type:	text/x-python
Branch:	MAIN
Log Message:	Added interface to DBS-2 which uses the new DBS-2 APIs. This reduces the number of api calls to 1.
#	User	Rev	Content
1	gutsche	1.1	#!/usr/bin/env python
2			import exceptions
3			import DBS2API.dbsApi
4			from DBS2API.dbsApiException import *
5			import common
6
7
8			# #######################################
9			class DBSError_DBS2(exceptions.Exception):
10			def __init__(self, errorName, errorMessage):
11			args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
12			exceptions.Exception.__init__(self, args)
13			pass
14
15			def getErrorMessage(self):
16			""" Return error message """
17			return "%s" % (self.args)
18
19			# #######################################
20			class DBSInvalidDataTierError_DBS2(exceptions.Exception):
21			def __init__(self, errorName, errorMessage):
22			args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
23			exceptions.Exception.__init__(self, args)
24			pass
25
26			def getErrorMessage(self):
27			""" Return error message """
28			return "%s" % (self.args)
29
30			# #######################################
31			class DBSInfoError_DBS2:
32			def __init__(self, url):
33			print '\nERROR accessing DBS url : '+url+'\n'
34			pass
35
36			# ####################################
37			class DataDiscoveryError_DBS2(exceptions.Exception):
38			def __init__(self, errorMessage):
39			self.args=errorMessage
40			exceptions.Exception.__init__(self, self.args)
41			pass
42
43			def getErrorMessage(self):
44			""" Return exception error """
45			return "%s" % (self.args)
46
47			# ####################################
48			class NotExistingDatasetError_DBS2(exceptions.Exception):
49			def __init__(self, errorMessage):
50			self.args=errorMessage
51			exceptions.Exception.__init__(self, self.args)
52			pass
53
54			def getErrorMessage(self):
55			""" Return exception error """
56			return "%s" % (self.args)
57
58			# ####################################
59			class NoDataTierinProvenanceError_DBS2(exceptions.Exception):
60			def __init__(self, errorMessage):
61			self.args=errorMessage
62			exceptions.Exception.__init__(self, self.args)
63			pass
64
65			def getErrorMessage(self):
66			""" Return exception error """
67			return "%s" % (self.args)
68
69			# ####################################
70			# class to find and extact info from published data
71			class DataDiscovery_DBS2:
72			def __init__(self, datasetPath, cfg_params):
73
74			# Attributes
75			self.datasetPath = datasetPath
76			self.cfg_params = cfg_params
77
78			self.eventsPerBlock = {} # DBS output: map fileblocks-events for collection
79			self.eventsPerFile = {} # DBS output: map files-events
80			self.blocksinfo = {} # DBS output: map fileblocks-files
81			self.maxEvents = 0 # DBS output: max events
82
83			# ####################################
84			def fetchDBSInfo(self):
85			"""
86			Contact DBS
87			"""
88
89			## get DBS URL
90			try:
91			dbs_url=self.cfg_params['CMSSW.dbs_url']
92			except KeyError:
93			dbs_url="http://cmsdoc.cern.ch/cms/test/aprom/DBS/CGIServer/prodquery"
94
95			## service API
96			args = {}
97			args['url']=dbs_url
98
99			common.logger.debug(3,"Accessing DBS at: "+dbs_url)
100
101			api = DBS2API.dbsApi.DbsApi(args)
102			try:
103			files = api.listFiles(self.datasetPath)
104			except DbsBadRequest, msg:
105			raise DataDiscoveryError(msg)
106			except DBSError_DBS2, msg:
107			raise DataDiscoveryError(msg)
108
109			# parse files and fill arrays
110			for file in files :
111			filename = file['LogicalFileName']
112			events = file['NumberOfEvents']
113			fileblock = file['Block']['Name']
114
115			# number of events per block
116			if fileblock in self.eventsPerBlock.keys() :
117			self.eventsPerBlock[fileblock] += events
118			else :
119			self.eventsPerBlock[fileblock] = events
120
121			# number of events per file
122			self.eventsPerFile[filename] = events
123
124			# number of events per block
125			if fileblock in self.blocksinfo.keys() :
126			self.blocksinfo[fileblock].append(filename)
127			else :
128			self.blocksinfo[fileblock] = [filename]
129
130			# total number of events
131			self.maxEvents += events
132
133			if len(self.eventsPerBlock) <= 0:
134			raise NotExistingDatasetError (("\nNo data for %s in DBS\nPlease check"
135			+ " dataset path variables in crab.cfg")
136			% self.datasetPath)
137
138
139			# #################################################
140			def getMaxEvents(self):
141			"""
142			max events
143			"""
144			return self.maxEvents
145
146			# #################################################
147			def getEventsPerBlock(self):
148			"""
149			list the event collections structure by fileblock
150			"""
151			return self.eventsPerBlock
152
153			# #################################################
154			def getEventsPerFile(self):
155			"""
156			list the event collections structure by file
157			"""
158			return self.eventsPerFile
159
160			# #################################################
161			def getFiles(self):
162			"""
163			return files grouped by fileblock
164			"""
165			return self.blocksinfo
166
167			########################################################################