CRAB/python/DataDiscovery_DBS2.py

#!/usr/bin/env python
import exceptions
import DBSAPI.dbsApi
from DBSAPI.dbsApiException import * 
import common


# #######################################
class DBSError_DBS2(exceptions.Exception):
    def __init__(self, errorName, errorMessage):
        args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
        exceptions.Exception.__init__(self, args)
        pass
    
    def getErrorMessage(self):
        """ Return error message """
        return "%s" % (self.args)

# #######################################
class DBSInvalidDataTierError_DBS2(exceptions.Exception):
    def __init__(self, errorName, errorMessage):
        args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
        exceptions.Exception.__init__(self, args)
        pass
    
    def getErrorMessage(self):
        """ Return error message """
        return "%s" % (self.args)

# #######################################
class DBSInfoError_DBS2:
    def __init__(self, url):
        print '\nERROR accessing DBS url : '+url+'\n'
        pass

# ####################################
class DataDiscoveryError_DBS2(exceptions.Exception):
    def __init__(self, errorMessage):
        self.args=errorMessage
        exceptions.Exception.__init__(self, self.args)
        pass

    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
class NotExistingDatasetError_DBS2(exceptions.Exception):
    def __init__(self, errorMessage):
        self.args=errorMessage
        exceptions.Exception.__init__(self, self.args)
        pass

    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
class NoDataTierinProvenanceError_DBS2(exceptions.Exception):
    def __init__(self, errorMessage):
        self.args=errorMessage
        exceptions.Exception.__init__(self, self.args)
        pass

    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
# class to find and extact info from published data
class DataDiscovery_DBS2:
    def __init__(self, datasetPath, cfg_params):

        #       Attributes
        self.datasetPath = datasetPath
        self.cfg_params = cfg_params

        self.eventsPerBlock = {}  # DBS output: map fileblocks-events for collection
        self.eventsPerFile = {}   # DBS output: map files-events
        self.blocksinfo = {}      # DBS output: map fileblocks-files 
        self.maxEvents = 0        # DBS output: max events

# ####################################
    def fetchDBSInfo(self):
        """
        Contact DBS
        """

        ## get DBS URL
        try:
            dbs_url=self.cfg_params['CMSSW.dbs_url']
        except KeyError:
            dbs_url="http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"

        ## get DBS URL
        try:
            dbs_version=self.cfg_params['CMSSW.dbs_version']
        except KeyError:
            dbs_version="v00_00_06"

        common.logger.debug(3,"Accessing DBS at: "+dbs_url+" with version: "+dbs_version)


        ## service API
        args = {}
        args['url']     = dbs_url
        args['version'] = dbs_version
        args['level']   = 'CRITICAL'

        api = DBSAPI.dbsApi.DbsApi(args)
        try:
            files = api.listFiles(self.datasetPath)
        except DbsBadRequest, msg:
            raise DataDiscoveryError_DBS2(msg)
        except DBSError_DBS2, msg:
            raise DataDiscoveryError_DBS2(msg)
        
        # parse files and fill arrays
        for file in files :
            filename = file['LogicalFileName']
            events = file['NumberOfEvents']
            fileblock = file['Block']['Name']

            # number of events per block
            if fileblock in self.eventsPerBlock.keys() :
                self.eventsPerBlock[fileblock] += events
            else :
                self.eventsPerBlock[fileblock] = events

            # number of events per file
            self.eventsPerFile[filename] = events

            # number of events per block
            if fileblock in self.blocksinfo.keys() :
                self.blocksinfo[fileblock].append(filename)
            else :
                self.blocksinfo[fileblock] = [filename]

            # total number of events
            self.maxEvents += events

        for block in self.eventsPerBlock.keys() :
            common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(self.eventsPerBlock[block],block))

        if len(self.eventsPerBlock) <= 0:
            raise NotExistingDatasetError_DBS2 (("\nNo data for %s in DBS\nPlease check"
                                            + " dataset path variables in crab.cfg")
                                            % self.datasetPath)


# #################################################
    def getMaxEvents(self):
        """
        max events 
        """
        return self.maxEvents

# #################################################
    def getEventsPerBlock(self):
        """
        list the event collections structure by fileblock 
        """
        return self.eventsPerBlock

# #################################################
    def getEventsPerFile(self):
        """
        list the event collections structure by file 
        """
        return self.eventsPerFile

# #################################################
    def getFiles(self):
        """
        return files grouped by fileblock 
        """
        return self.blocksinfo        

########################################################################
Revision:	1.7
Committed:	Thu Apr 26 19:26:28 2007 UTC (18 years ago) by gutsche
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_2_0_0_pre3, CRAB_1_5_2, CRAB_2_0_0_pre2, CRAB_2_0_0_pre1, CRAB_1_5_1, CRAB_1_5_1_pre4, CRAB_1_5_1_pre3, CRAB_1_5_1_pre2, CRAB_1_5_1_pre1
Changes since 1.6:	+6 -0 lines
Log Message:	add debug output similar to DBS-1 data discovery
#	User	Rev	Content
1	gutsche	1.1	#!/usr/bin/env python
2			import exceptions
3	gutsche	1.2	import DBSAPI.dbsApi
4			from DBSAPI.dbsApiException import *
5	gutsche	1.1	import common
6
7
8			# #######################################
9			class DBSError_DBS2(exceptions.Exception):
10			def __init__(self, errorName, errorMessage):
11			args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
12			exceptions.Exception.__init__(self, args)
13			pass
14
15			def getErrorMessage(self):
16			""" Return error message """
17			return "%s" % (self.args)
18
19			# #######################################
20			class DBSInvalidDataTierError_DBS2(exceptions.Exception):
21			def __init__(self, errorName, errorMessage):
22			args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
23			exceptions.Exception.__init__(self, args)
24			pass
25
26			def getErrorMessage(self):
27			""" Return error message """
28			return "%s" % (self.args)
29
30			# #######################################
31			class DBSInfoError_DBS2:
32			def __init__(self, url):
33			print '\nERROR accessing DBS url : '+url+'\n'
34			pass
35
36			# ####################################
37			class DataDiscoveryError_DBS2(exceptions.Exception):
38			def __init__(self, errorMessage):
39			self.args=errorMessage
40			exceptions.Exception.__init__(self, self.args)
41			pass
42
43			def getErrorMessage(self):
44			""" Return exception error """
45			return "%s" % (self.args)
46
47			# ####################################
48			class NotExistingDatasetError_DBS2(exceptions.Exception):
49			def __init__(self, errorMessage):
50			self.args=errorMessage
51			exceptions.Exception.__init__(self, self.args)
52			pass
53
54			def getErrorMessage(self):
55			""" Return exception error """
56			return "%s" % (self.args)
57
58			# ####################################
59			class NoDataTierinProvenanceError_DBS2(exceptions.Exception):
60			def __init__(self, errorMessage):
61			self.args=errorMessage
62			exceptions.Exception.__init__(self, self.args)
63			pass
64
65			def getErrorMessage(self):
66			""" Return exception error """
67			return "%s" % (self.args)
68
69			# ####################################
70			# class to find and extact info from published data
71			class DataDiscovery_DBS2:
72			def __init__(self, datasetPath, cfg_params):
73
74			# Attributes
75			self.datasetPath = datasetPath
76			self.cfg_params = cfg_params
77
78			self.eventsPerBlock = {} # DBS output: map fileblocks-events for collection
79			self.eventsPerFile = {} # DBS output: map files-events
80			self.blocksinfo = {} # DBS output: map fileblocks-files
81			self.maxEvents = 0 # DBS output: max events
82
83			# ####################################
84			def fetchDBSInfo(self):
85			"""
86			Contact DBS
87			"""
88
89			## get DBS URL
90			try:
91			dbs_url=self.cfg_params['CMSSW.dbs_url']
92			except KeyError:
93	gutsche	1.6	dbs_url="http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
94	gutsche	1.1
95	gutsche	1.3	## get DBS URL
96			try:
97			dbs_version=self.cfg_params['CMSSW.dbs_version']
98			except KeyError:
99	gutsche	1.5	dbs_version="v00_00_06"
100	gutsche	1.3
101	gutsche	1.7	common.logger.debug(3,"Accessing DBS at: "+dbs_url+" with version: "+dbs_version)
102
103
104	gutsche	1.1	## service API
105			args = {}
106	gutsche	1.3	args['url'] = dbs_url
107			args['version'] = dbs_version
108	gutsche	1.4	args['level'] = 'CRITICAL'
109	gutsche	1.1
110	gutsche	1.2	api = DBSAPI.dbsApi.DbsApi(args)
111	gutsche	1.1	try:
112			files = api.listFiles(self.datasetPath)
113			except DbsBadRequest, msg:
114	gutsche	1.2	raise DataDiscoveryError_DBS2(msg)
115	gutsche	1.1	except DBSError_DBS2, msg:
116	gutsche	1.2	raise DataDiscoveryError_DBS2(msg)
117	gutsche	1.1
118			# parse files and fill arrays
119			for file in files :
120			filename = file['LogicalFileName']
121			events = file['NumberOfEvents']
122			fileblock = file['Block']['Name']
123
124			# number of events per block
125			if fileblock in self.eventsPerBlock.keys() :
126			self.eventsPerBlock[fileblock] += events
127			else :
128			self.eventsPerBlock[fileblock] = events
129
130			# number of events per file
131			self.eventsPerFile[filename] = events
132
133			# number of events per block
134			if fileblock in self.blocksinfo.keys() :
135			self.blocksinfo[fileblock].append(filename)
136			else :
137			self.blocksinfo[fileblock] = [filename]
138
139			# total number of events
140			self.maxEvents += events
141
142	gutsche	1.7	for block in self.eventsPerBlock.keys() :
143			common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(self.eventsPerBlock[block],block))
144
145	gutsche	1.1	if len(self.eventsPerBlock) <= 0:
146	gutsche	1.2	raise NotExistingDatasetError_DBS2 (("\nNo data for %s in DBS\nPlease check"
147	gutsche	1.1	+ " dataset path variables in crab.cfg")
148			% self.datasetPath)
149
150
151			# #################################################
152			def getMaxEvents(self):
153			"""
154			max events
155			"""
156			return self.maxEvents
157
158			# #################################################
159			def getEventsPerBlock(self):
160			"""
161			list the event collections structure by fileblock
162			"""
163			return self.eventsPerBlock
164
165			# #################################################
166			def getEventsPerFile(self):
167			"""
168			list the event collections structure by file
169			"""
170			return self.eventsPerFile
171
172			# #################################################
173			def getFiles(self):
174			"""
175			return files grouped by fileblock
176			"""
177			return self.blocksinfo
178
179			########################################################################