CRAB/python/DataDiscovery_DBS2.py

#!/usr/bin/env python
import exceptions
import DBSAPI.dbsApi
from DBSAPI.dbsApiException import * 
import common


# #######################################
class DBSError_DBS2(exceptions.Exception):
    def __init__(self, errorName, errorMessage):
        args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
        exceptions.Exception.__init__(self, args)
        pass
    
    def getErrorMessage(self):
        """ Return error message """
        return "%s" % (self.args)

# #######################################
class DBSInvalidDataTierError_DBS2(exceptions.Exception):
    def __init__(self, errorName, errorMessage):
        args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
        exceptions.Exception.__init__(self, args)
        pass
    
    def getErrorMessage(self):
        """ Return error message """
        return "%s" % (self.args)

# #######################################
class DBSInfoError_DBS2:
    def __init__(self, url):
        print '\nERROR accessing DBS url : '+url+'\n'
        pass

# ####################################
class DataDiscoveryError_DBS2(exceptions.Exception):
    def __init__(self, errorMessage):
        self.args=errorMessage
        exceptions.Exception.__init__(self, self.args)
        pass

    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
class NotExistingDatasetError_DBS2(exceptions.Exception):
    def __init__(self, errorMessage):
        self.args=errorMessage
        exceptions.Exception.__init__(self, self.args)
        pass

    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
class NoDataTierinProvenanceError_DBS2(exceptions.Exception):
    def __init__(self, errorMessage):
        self.args=errorMessage
        exceptions.Exception.__init__(self, self.args)
        pass

    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
# class to find and extact info from published data
class DataDiscovery_DBS2:
    def __init__(self, datasetPath, cfg_params):

        #       Attributes
        self.datasetPath = datasetPath
        self.cfg_params = cfg_params

        self.eventsPerBlock = {}  # DBS output: map fileblocks-events for collection
        self.eventsPerFile = {}   # DBS output: map files-events
        self.blocksinfo = {}      # DBS output: map fileblocks-files 
        self.maxEvents = 0        # DBS output: max events

# ####################################
    def fetchDBSInfo(self):
        """
        Contact DBS
        """

        ## get DBS URL
        try:
            dbs_url=self.cfg_params['CMSSW.dbs_url']
        except KeyError:
            dbs_url="http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"

        common.logger.debug(3,"Accessing DBS at: "+dbs_url)


        ## service API
        args = {}
        args['url']     = dbs_url
        args['level']   = 'CRITICAL'

        api = DBSAPI.dbsApi.DbsApi(args)
        try:
            files = api.listDatasetFiles(self.datasetPath)
        except DbsBadRequest, msg:
            raise DataDiscoveryError_DBS2(msg)
        except DBSError_DBS2, msg:
            raise DataDiscoveryError_DBS2(msg)

        # parse files and fill arrays
        for file in files :
            filename = file['LogicalFileName']
            events = file['NumberOfEvents']
            fileblock = file['Block']['Name']

            # number of events per block
            if fileblock in self.eventsPerBlock.keys() :
                self.eventsPerBlock[fileblock] += events
            else :
                self.eventsPerBlock[fileblock] = events

            # number of events per file
            self.eventsPerFile[filename] = events

            # number of events per block
            if fileblock in self.blocksinfo.keys() :
                self.blocksinfo[fileblock].append(filename)
            else :
                self.blocksinfo[fileblock] = [filename]

            # total number of events
            self.maxEvents += events

        for block in self.eventsPerBlock.keys() :
            common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(self.eventsPerBlock[block],block))

        if len(self.eventsPerBlock) <= 0:
            raise NotExistingDatasetError_DBS2 (("\nNo data for %s in DBS\nPlease check"
                                            + " dataset path variables in crab.cfg")
                                            % self.datasetPath)


# #################################################
    def getMaxEvents(self):
        """
        max events 
        """
        return self.maxEvents

# #################################################
    def getEventsPerBlock(self):
        """
        list the event collections structure by fileblock 
        """
        return self.eventsPerBlock

# #################################################
    def getEventsPerFile(self):
        """
        list the event collections structure by file 
        """
        return self.eventsPerFile

# #################################################
    def getFiles(self):
        """
        return files grouped by fileblock 
        """
        return self.blocksinfo        

########################################################################
Revision:	1.8
Committed:	Mon Jun 18 15:34:47 2007 UTC (17 years, 10 months ago) by gutsche
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_1_5_3_pre2, CRAB_1_5_3_pre1
Changes since 1.7:	+3 -10 lines
Log Message:	changed to new DBS api which treats PDS and ADS the same, updated DBS tag to 1_0_4_pre2, removed card for dbs version in crab.cfg
#	User	Rev	Content
1	gutsche	1.1	#!/usr/bin/env python
2			import exceptions
3	gutsche	1.2	import DBSAPI.dbsApi
4			from DBSAPI.dbsApiException import *
5	gutsche	1.1	import common
6
7
8			# #######################################
9			class DBSError_DBS2(exceptions.Exception):
10			def __init__(self, errorName, errorMessage):
11			args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
12			exceptions.Exception.__init__(self, args)
13			pass
14
15			def getErrorMessage(self):
16			""" Return error message """
17			return "%s" % (self.args)
18
19			# #######################################
20			class DBSInvalidDataTierError_DBS2(exceptions.Exception):
21			def __init__(self, errorName, errorMessage):
22			args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
23			exceptions.Exception.__init__(self, args)
24			pass
25
26			def getErrorMessage(self):
27			""" Return error message """
28			return "%s" % (self.args)
29
30			# #######################################
31			class DBSInfoError_DBS2:
32			def __init__(self, url):
33			print '\nERROR accessing DBS url : '+url+'\n'
34			pass
35
36			# ####################################
37			class DataDiscoveryError_DBS2(exceptions.Exception):
38			def __init__(self, errorMessage):
39			self.args=errorMessage
40			exceptions.Exception.__init__(self, self.args)
41			pass
42
43			def getErrorMessage(self):
44			""" Return exception error """
45			return "%s" % (self.args)
46
47			# ####################################
48			class NotExistingDatasetError_DBS2(exceptions.Exception):
49			def __init__(self, errorMessage):
50			self.args=errorMessage
51			exceptions.Exception.__init__(self, self.args)
52			pass
53
54			def getErrorMessage(self):
55			""" Return exception error """
56			return "%s" % (self.args)
57
58			# ####################################
59			class NoDataTierinProvenanceError_DBS2(exceptions.Exception):
60			def __init__(self, errorMessage):
61			self.args=errorMessage
62			exceptions.Exception.__init__(self, self.args)
63			pass
64
65			def getErrorMessage(self):
66			""" Return exception error """
67			return "%s" % (self.args)
68
69			# ####################################
70			# class to find and extact info from published data
71			class DataDiscovery_DBS2:
72			def __init__(self, datasetPath, cfg_params):
73
74			# Attributes
75			self.datasetPath = datasetPath
76			self.cfg_params = cfg_params
77
78			self.eventsPerBlock = {} # DBS output: map fileblocks-events for collection
79			self.eventsPerFile = {} # DBS output: map files-events
80			self.blocksinfo = {} # DBS output: map fileblocks-files
81			self.maxEvents = 0 # DBS output: max events
82
83			# ####################################
84			def fetchDBSInfo(self):
85			"""
86			Contact DBS
87			"""
88
89			## get DBS URL
90			try:
91			dbs_url=self.cfg_params['CMSSW.dbs_url']
92			except KeyError:
93	gutsche	1.6	dbs_url="http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
94	gutsche	1.1
95	gutsche	1.8	common.logger.debug(3,"Accessing DBS at: "+dbs_url)
96	gutsche	1.7
97
98	gutsche	1.1	## service API
99			args = {}
100	gutsche	1.3	args['url'] = dbs_url
101	gutsche	1.4	args['level'] = 'CRITICAL'
102	gutsche	1.1
103	gutsche	1.2	api = DBSAPI.dbsApi.DbsApi(args)
104	gutsche	1.1	try:
105	gutsche	1.8	files = api.listDatasetFiles(self.datasetPath)
106	gutsche	1.1	except DbsBadRequest, msg:
107	gutsche	1.2	raise DataDiscoveryError_DBS2(msg)
108	gutsche	1.1	except DBSError_DBS2, msg:
109	gutsche	1.2	raise DataDiscoveryError_DBS2(msg)
110	gutsche	1.8
111	gutsche	1.1	# parse files and fill arrays
112			for file in files :
113			filename = file['LogicalFileName']
114			events = file['NumberOfEvents']
115			fileblock = file['Block']['Name']
116
117			# number of events per block
118			if fileblock in self.eventsPerBlock.keys() :
119			self.eventsPerBlock[fileblock] += events
120			else :
121			self.eventsPerBlock[fileblock] = events
122
123			# number of events per file
124			self.eventsPerFile[filename] = events
125
126			# number of events per block
127			if fileblock in self.blocksinfo.keys() :
128			self.blocksinfo[fileblock].append(filename)
129			else :
130			self.blocksinfo[fileblock] = [filename]
131
132			# total number of events
133			self.maxEvents += events
134
135	gutsche	1.7	for block in self.eventsPerBlock.keys() :
136			common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(self.eventsPerBlock[block],block))
137
138	gutsche	1.1	if len(self.eventsPerBlock) <= 0:
139	gutsche	1.2	raise NotExistingDatasetError_DBS2 (("\nNo data for %s in DBS\nPlease check"
140	gutsche	1.1	+ " dataset path variables in crab.cfg")
141			% self.datasetPath)
142
143
144			# #################################################
145			def getMaxEvents(self):
146			"""
147			max events
148			"""
149			return self.maxEvents
150
151			# #################################################
152			def getEventsPerBlock(self):
153			"""
154			list the event collections structure by fileblock
155			"""
156			return self.eventsPerBlock
157
158			# #################################################
159			def getEventsPerFile(self):
160			"""
161			list the event collections structure by file
162			"""
163			return self.eventsPerFile
164
165			# #################################################
166			def getFiles(self):
167			"""
168			return files grouped by fileblock
169			"""
170			return self.blocksinfo
171
172			########################################################################