CRAB/python/DataDiscovery_DBS2.py

#!/usr/bin/env python
import exceptions
import DBSAPI.dbsApi
from DBSAPI.dbsApiException import * 
import common


# #######################################
class DBSError_DBS2(exceptions.Exception):
    def __init__(self, errorName, errorMessage):
        args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
        exceptions.Exception.__init__(self, args)
        pass
    
    def getErrorMessage(self):
        """ Return error message """
        return "%s" % (self.args)

# #######################################
class DBSInvalidDataTierError_DBS2(exceptions.Exception):
    def __init__(self, errorName, errorMessage):
        args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
        exceptions.Exception.__init__(self, args)
        pass
    
    def getErrorMessage(self):
        """ Return error message """
        return "%s" % (self.args)

# #######################################
class DBSInfoError_DBS2:
    def __init__(self, url):
        print '\nERROR accessing DBS url : '+url+'\n'
        pass

# ####################################
class DataDiscoveryError_DBS2(exceptions.Exception):
    def __init__(self, errorMessage):
        self.args=errorMessage
        exceptions.Exception.__init__(self, self.args)
        pass

    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
class NotExistingDatasetError_DBS2(exceptions.Exception):
    def __init__(self, errorMessage):
        self.args=errorMessage
        exceptions.Exception.__init__(self, self.args)
        pass

    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
class NoDataTierinProvenanceError_DBS2(exceptions.Exception):
    def __init__(self, errorMessage):
        self.args=errorMessage
        exceptions.Exception.__init__(self, self.args)
        pass

    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
# class to find and extact info from published data
class DataDiscovery_DBS2:
    def __init__(self, datasetPath, cfg_params):

        #       Attributes
        self.datasetPath = datasetPath
        self.cfg_params = cfg_params

        self.eventsPerBlock = {}  # DBS output: map fileblocks-events for collection
        self.eventsPerFile = {}   # DBS output: map files-events
        self.blocksinfo = {}      # DBS output: map fileblocks-files 
        self.maxEvents = 0        # DBS output: max events

# ####################################
    def fetchDBSInfo(self):
        """
        Contact DBS
        """

        ## get DBS URL
        try:
            dbs_url=self.cfg_params['CMSSW.dbs_url']
        except KeyError:
            dbs_url="http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"

        ## get DBS URL
        try:
            dbs_version=self.cfg_params['CMSSW.dbs_version']
        except KeyError:
            dbs_version="v00_00_06"

        ## service API
        args = {}
        args['url']     = dbs_url
        args['version'] = dbs_version
        args['level']   = 'CRITICAL'

        api = DBSAPI.dbsApi.DbsApi(args)
        try:
            files = api.listFiles(self.datasetPath)
        except DbsBadRequest, msg:
            raise DataDiscoveryError_DBS2(msg)
        except DBSError_DBS2, msg:
            raise DataDiscoveryError_DBS2(msg)
        
        # parse files and fill arrays
        for file in files :
            filename = file['LogicalFileName']
            events = file['NumberOfEvents']
            fileblock = file['Block']['Name']

            # number of events per block
            if fileblock in self.eventsPerBlock.keys() :
                self.eventsPerBlock[fileblock] += events
            else :
                self.eventsPerBlock[fileblock] = events

            # number of events per file
            self.eventsPerFile[filename] = events

            # number of events per block
            if fileblock in self.blocksinfo.keys() :
                self.blocksinfo[fileblock].append(filename)
            else :
                self.blocksinfo[fileblock] = [filename]

            # total number of events
            self.maxEvents += events

        if len(self.eventsPerBlock) <= 0:
            raise NotExistingDatasetError_DBS2 (("\nNo data for %s in DBS\nPlease check"
                                            + " dataset path variables in crab.cfg")
                                            % self.datasetPath)


# #################################################
    def getMaxEvents(self):
        """
        max events 
        """
        return self.maxEvents

# #################################################
    def getEventsPerBlock(self):
        """
        list the event collections structure by fileblock 
        """
        return self.eventsPerBlock

# #################################################
    def getEventsPerFile(self):
        """
        list the event collections structure by file 
        """
        return self.eventsPerFile

# #################################################
    def getFiles(self):
        """
        return files grouped by fileblock 
        """
        return self.blocksinfo        

########################################################################
Revision:	1.6
Committed:	Thu Apr 26 16:38:08 2007 UTC (18 years ago) by gutsche
Content type:	text/x-python
Branch:	MAIN
Changes since 1.5:	+1 -1 lines
Log Message:	new default for DBS-2
#	User	Rev	Content
1	gutsche	1.1	#!/usr/bin/env python
2			import exceptions
3	gutsche	1.2	import DBSAPI.dbsApi
4			from DBSAPI.dbsApiException import *
5	gutsche	1.1	import common
6
7
8			# #######################################
9			class DBSError_DBS2(exceptions.Exception):
10			def __init__(self, errorName, errorMessage):
11			args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
12			exceptions.Exception.__init__(self, args)
13			pass
14
15			def getErrorMessage(self):
16			""" Return error message """
17			return "%s" % (self.args)
18
19			# #######################################
20			class DBSInvalidDataTierError_DBS2(exceptions.Exception):
21			def __init__(self, errorName, errorMessage):
22			args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
23			exceptions.Exception.__init__(self, args)
24			pass
25
26			def getErrorMessage(self):
27			""" Return error message """
28			return "%s" % (self.args)
29
30			# #######################################
31			class DBSInfoError_DBS2:
32			def __init__(self, url):
33			print '\nERROR accessing DBS url : '+url+'\n'
34			pass
35
36			# ####################################
37			class DataDiscoveryError_DBS2(exceptions.Exception):
38			def __init__(self, errorMessage):
39			self.args=errorMessage
40			exceptions.Exception.__init__(self, self.args)
41			pass
42
43			def getErrorMessage(self):
44			""" Return exception error """
45			return "%s" % (self.args)
46
47			# ####################################
48			class NotExistingDatasetError_DBS2(exceptions.Exception):
49			def __init__(self, errorMessage):
50			self.args=errorMessage
51			exceptions.Exception.__init__(self, self.args)
52			pass
53
54			def getErrorMessage(self):
55			""" Return exception error """
56			return "%s" % (self.args)
57
58			# ####################################
59			class NoDataTierinProvenanceError_DBS2(exceptions.Exception):
60			def __init__(self, errorMessage):
61			self.args=errorMessage
62			exceptions.Exception.__init__(self, self.args)
63			pass
64
65			def getErrorMessage(self):
66			""" Return exception error """
67			return "%s" % (self.args)
68
69			# ####################################
70			# class to find and extact info from published data
71			class DataDiscovery_DBS2:
72			def __init__(self, datasetPath, cfg_params):
73
74			# Attributes
75			self.datasetPath = datasetPath
76			self.cfg_params = cfg_params
77
78			self.eventsPerBlock = {} # DBS output: map fileblocks-events for collection
79			self.eventsPerFile = {} # DBS output: map files-events
80			self.blocksinfo = {} # DBS output: map fileblocks-files
81			self.maxEvents = 0 # DBS output: max events
82
83			# ####################################
84			def fetchDBSInfo(self):
85			"""
86			Contact DBS
87			"""
88
89			## get DBS URL
90			try:
91			dbs_url=self.cfg_params['CMSSW.dbs_url']
92			except KeyError:
93	gutsche	1.6	dbs_url="http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
94	gutsche	1.1
95	gutsche	1.3	## get DBS URL
96			try:
97			dbs_version=self.cfg_params['CMSSW.dbs_version']
98			except KeyError:
99	gutsche	1.5	dbs_version="v00_00_06"
100	gutsche	1.3
101	gutsche	1.1	## service API
102			args = {}
103	gutsche	1.3	args['url'] = dbs_url
104			args['version'] = dbs_version
105	gutsche	1.4	args['level'] = 'CRITICAL'
106	gutsche	1.1
107	gutsche	1.2	api = DBSAPI.dbsApi.DbsApi(args)
108	gutsche	1.1	try:
109			files = api.listFiles(self.datasetPath)
110			except DbsBadRequest, msg:
111	gutsche	1.2	raise DataDiscoveryError_DBS2(msg)
112	gutsche	1.1	except DBSError_DBS2, msg:
113	gutsche	1.2	raise DataDiscoveryError_DBS2(msg)
114	gutsche	1.1
115			# parse files and fill arrays
116			for file in files :
117			filename = file['LogicalFileName']
118			events = file['NumberOfEvents']
119			fileblock = file['Block']['Name']
120
121			# number of events per block
122			if fileblock in self.eventsPerBlock.keys() :
123			self.eventsPerBlock[fileblock] += events
124			else :
125			self.eventsPerBlock[fileblock] = events
126
127			# number of events per file
128			self.eventsPerFile[filename] = events
129
130			# number of events per block
131			if fileblock in self.blocksinfo.keys() :
132			self.blocksinfo[fileblock].append(filename)
133			else :
134			self.blocksinfo[fileblock] = [filename]
135
136			# total number of events
137			self.maxEvents += events
138
139			if len(self.eventsPerBlock) <= 0:
140	gutsche	1.2	raise NotExistingDatasetError_DBS2 (("\nNo data for %s in DBS\nPlease check"
141	gutsche	1.1	+ " dataset path variables in crab.cfg")
142			% self.datasetPath)
143
144
145			# #################################################
146			def getMaxEvents(self):
147			"""
148			max events
149			"""
150			return self.maxEvents
151
152			# #################################################
153			def getEventsPerBlock(self):
154			"""
155			list the event collections structure by fileblock
156			"""
157			return self.eventsPerBlock
158
159			# #################################################
160			def getEventsPerFile(self):
161			"""
162			list the event collections structure by file
163			"""
164			return self.eventsPerFile
165
166			# #################################################
167			def getFiles(self):
168			"""
169			return files grouped by fileblock
170			"""
171			return self.blocksinfo
172
173			########################################################################