CRAB/python/DataDiscovery_DBS2.py

#!/usr/bin/env python
import exceptions
import DBSAPI.dbsApi
from DBSAPI.dbsApiException import * 
import common
from crab_util import *


# #######################################
class DBSError_DBS2(exceptions.Exception):
    def __init__(self, errorName, errorMessage):
        args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
        exceptions.Exception.__init__(self, args)
        pass
    
    def getErrorMessage(self):
        """ Return error message """
        return "%s" % (self.args)

# #######################################
class DBSInvalidDataTierError_DBS2(exceptions.Exception):
    def __init__(self, errorName, errorMessage):
        args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
        exceptions.Exception.__init__(self, args)
        pass
    
    def getErrorMessage(self):
        """ Return error message """
        return "%s" % (self.args)

# #######################################
class DBSInfoError_DBS2:
    def __init__(self, url):
        print '\nERROR accessing DBS url : '+url+'\n'
        pass

# ####################################
class DataDiscoveryError_DBS2(exceptions.Exception):
    def __init__(self, errorMessage):
        self.args=errorMessage
        exceptions.Exception.__init__(self, self.args)
        pass

    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
class NotExistingDatasetError_DBS2(exceptions.Exception):
    def __init__(self, errorMessage):
        self.args=errorMessage
        exceptions.Exception.__init__(self, self.args)
        pass

    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
class NoDataTierinProvenanceError_DBS2(exceptions.Exception):
    def __init__(self, errorMessage):
        self.args=errorMessage
        exceptions.Exception.__init__(self, self.args)
        pass

    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
# class to find and extact info from published data
class DataDiscovery_DBS2:
    def __init__(self, datasetPath, cfg_params):

        #       Attributes
        self.datasetPath = datasetPath
        self.cfg_params = cfg_params

        self.eventsPerBlock = {}  # DBS output: map fileblocks-events for collection
        self.eventsPerFile = {}   # DBS output: map files-events
        self.blocksinfo = {}      # DBS output: map fileblocks-files 
        self.maxEvents = 0        # DBS output: max events

# ####################################
    def fetchDBSInfo(self):
        """
        Contact DBS
        """

        ## get DBS URL
        try:
            dbs_url=self.cfg_params['CMSSW.dbs_url']
        except KeyError:
            dbs_url="http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"

        common.logger.debug(3,"Accessing DBS at: "+dbs_url)

        ## check if runs are selected
        try:
            runselection = parseRange2(self.cfg_params['CMSSW.runselection'])
        except:
            runselection = []

        ## service API
        args = {}
        args['url']     = dbs_url
        args['level']   = 'CRITICAL'

        api = DBSAPI.dbsApi.DbsApi(args)
        try:
            if len(runselection) <= 0 :
                files = api.listDatasetFiles(self.datasetPath)
            else :
                files = api.listFiles(path=self.datasetPath, details=True)
        except DbsBadRequest, msg:
            raise DataDiscoveryError_DBS2(msg)
        except DBSError_DBS2, msg:
            raise DataDiscoveryError_DBS2(msg)

        # parse files and fill arrays
        for file in files :
            filename = file['LogicalFileName']
            if filename.find('.dat') < 0 :
                fileblock = file['Block']['Name']
                events    = file['NumberOfEvents']
                continue_flag = 0
                if len(runselection) > 0 :
                    runslist = file['RunsList']
                    for run in runslist :
                        runnumber = run['RunNumber']
                        for selected_run in runselection :
                            if runnumber == selected_run :
                                continue_flag = 1
                else :
                    continue_flag = 1

                if continue_flag == 1 :
                    # number of events per block
                    if fileblock in self.eventsPerBlock.keys() :
                        self.eventsPerBlock[fileblock] += events
                    else :
                        self.eventsPerBlock[fileblock] = events

                    # number of events per file
                    self.eventsPerFile[filename] = events

                    # number of events per block
                    if fileblock in self.blocksinfo.keys() :
                        self.blocksinfo[fileblock].append(filename)
                    else :
                        self.blocksinfo[fileblock] = [filename]

                    # total number of events
                    self.maxEvents += events

        for block in self.eventsPerBlock.keys() :
            common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(self.eventsPerBlock[block],block))

        if len(self.eventsPerBlock) <= 0:
            raise NotExistingDatasetError_DBS2 (("\nNo data for %s in DBS\nPlease check"
                                            + " dataset path variables in crab.cfg")
                                            % self.datasetPath)


# #################################################
    def getMaxEvents(self):
        """
        max events 
        """
        return self.maxEvents

# #################################################
    def getEventsPerBlock(self):
        """
        list the event collections structure by fileblock 
        """
        return self.eventsPerBlock

# #################################################
    def getEventsPerFile(self):
        """
        list the event collections structure by file 
        """
        return self.eventsPerFile

# #################################################
    def getFiles(self):
        """
        return files grouped by fileblock 
        """
        return self.blocksinfo        

########################################################################
Revision:	1.9
Committed:	Wed Jul 4 00:50:09 2007 UTC (17 years, 10 months ago) by gutsche
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_2_0_2_pre5, CRAB_2_0_2_pre4, CRAB_2_0_2_pre3, CRAB_1_5_4_SLC3, CRAB_1_5_4_SLC3_pre4, CRAB_2_0_2_pre2, CRAB_2_0_2_pre1, CRAB_1_5_4_SLC3_pre3, CRAB_2_0_1, CRAB_1_5_4_SLC3_pre2, CRAB_2_0_1_pre1, CRAB_1_5_4_SLC3_pre1, CRAB_2_0_0, CRAB_2_0_0_pre10, CRAB_2_0_0_pre9, CRAB_1_5_4, CRAB_1_5_4_pre2, CRAB_1_5_4_pre1, CRAB_2_0_0_pre7, CRAB_2_0_0_pre6, CRAB_1_5_3, CRAB_1_5_3_pre5, CRAB_1_5_3_pre4, CRAB_2_0_0_pre5, CRAB_1_5_3_pre3, configure, CRAB_2_0_0_pre4
Branch point for:	CRAB_1_5_4_SLC3_pre4_br, CRAB_1_5_4_SLC3_start
Changes since 1.8:	+41 -20 lines
Log Message:	Introduced run selection within single PDS crab.cfg card: runselection = 1,2,3-4 Parsing is done on UI, DBS has no suited API to do parsing on DBS server (yet) api.listFiles is used with details=True which increases query time significantly. In addition, filtered for .dat files which can be present in PDS mixed with .root
#	Content
1	#!/usr/bin/env python
2	import exceptions
3	import DBSAPI.dbsApi
4	from DBSAPI.dbsApiException import *
5	import common
6	from crab_util import *
7
8
9	# #######################################
10	class DBSError_DBS2(exceptions.Exception):
11	def __init__(self, errorName, errorMessage):
12	args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
13	exceptions.Exception.__init__(self, args)
14	pass
15
16	def getErrorMessage(self):
17	""" Return error message """
18	return "%s" % (self.args)
19
20	# #######################################
21	class DBSInvalidDataTierError_DBS2(exceptions.Exception):
22	def __init__(self, errorName, errorMessage):
23	args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
24	exceptions.Exception.__init__(self, args)
25	pass
26
27	def getErrorMessage(self):
28	""" Return error message """
29	return "%s" % (self.args)
30
31	# #######################################
32	class DBSInfoError_DBS2:
33	def __init__(self, url):
34	print '\nERROR accessing DBS url : '+url+'\n'
35	pass
36
37	# ####################################
38	class DataDiscoveryError_DBS2(exceptions.Exception):
39	def __init__(self, errorMessage):
40	self.args=errorMessage
41	exceptions.Exception.__init__(self, self.args)
42	pass
43
44	def getErrorMessage(self):
45	""" Return exception error """
46	return "%s" % (self.args)
47
48	# ####################################
49	class NotExistingDatasetError_DBS2(exceptions.Exception):
50	def __init__(self, errorMessage):
51	self.args=errorMessage
52	exceptions.Exception.__init__(self, self.args)
53	pass
54
55	def getErrorMessage(self):
56	""" Return exception error """
57	return "%s" % (self.args)
58
59	# ####################################
60	class NoDataTierinProvenanceError_DBS2(exceptions.Exception):
61	def __init__(self, errorMessage):
62	self.args=errorMessage
63	exceptions.Exception.__init__(self, self.args)
64	pass
65
66	def getErrorMessage(self):
67	""" Return exception error """
68	return "%s" % (self.args)
69
70	# ####################################
71	# class to find and extact info from published data
72	class DataDiscovery_DBS2:
73	def __init__(self, datasetPath, cfg_params):
74
75	# Attributes
76	self.datasetPath = datasetPath
77	self.cfg_params = cfg_params
78
79	self.eventsPerBlock = {} # DBS output: map fileblocks-events for collection
80	self.eventsPerFile = {} # DBS output: map files-events
81	self.blocksinfo = {} # DBS output: map fileblocks-files
82	self.maxEvents = 0 # DBS output: max events
83
84	# ####################################
85	def fetchDBSInfo(self):
86	"""
87	Contact DBS
88	"""
89
90	## get DBS URL
91	try:
92	dbs_url=self.cfg_params['CMSSW.dbs_url']
93	except KeyError:
94	dbs_url="http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
95
96	common.logger.debug(3,"Accessing DBS at: "+dbs_url)
97
98	## check if runs are selected
99	try:
100	runselection = parseRange2(self.cfg_params['CMSSW.runselection'])
101	except:
102	runselection = []
103
104	## service API
105	args = {}
106	args['url'] = dbs_url
107	args['level'] = 'CRITICAL'
108
109	api = DBSAPI.dbsApi.DbsApi(args)
110	try:
111	if len(runselection) <= 0 :
112	files = api.listDatasetFiles(self.datasetPath)
113	else :
114	files = api.listFiles(path=self.datasetPath, details=True)
115	except DbsBadRequest, msg:
116	raise DataDiscoveryError_DBS2(msg)
117	except DBSError_DBS2, msg:
118	raise DataDiscoveryError_DBS2(msg)
119
120	# parse files and fill arrays
121	for file in files :
122	filename = file['LogicalFileName']
123	if filename.find('.dat') < 0 :
124	fileblock = file['Block']['Name']
125	events = file['NumberOfEvents']
126	continue_flag = 0
127	if len(runselection) > 0 :
128	runslist = file['RunsList']
129	for run in runslist :
130	runnumber = run['RunNumber']
131	for selected_run in runselection :
132	if runnumber == selected_run :
133	continue_flag = 1
134	else :
135	continue_flag = 1
136
137	if continue_flag == 1 :
138	# number of events per block
139	if fileblock in self.eventsPerBlock.keys() :
140	self.eventsPerBlock[fileblock] += events
141	else :
142	self.eventsPerBlock[fileblock] = events
143
144	# number of events per file
145	self.eventsPerFile[filename] = events
146
147	# number of events per block
148	if fileblock in self.blocksinfo.keys() :
149	self.blocksinfo[fileblock].append(filename)
150	else :
151	self.blocksinfo[fileblock] = [filename]
152
153	# total number of events
154	self.maxEvents += events
155
156	for block in self.eventsPerBlock.keys() :
157	common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(self.eventsPerBlock[block],block))
158
159	if len(self.eventsPerBlock) <= 0:
160	raise NotExistingDatasetError_DBS2 (("\nNo data for %s in DBS\nPlease check"
161	+ " dataset path variables in crab.cfg")
162	% self.datasetPath)
163
164
165	# #################################################
166	def getMaxEvents(self):
167	"""
168	max events
169	"""
170	return self.maxEvents
171
172	# #################################################
173	def getEventsPerBlock(self):
174	"""
175	list the event collections structure by fileblock
176	"""
177	return self.eventsPerBlock
178
179	# #################################################
180	def getEventsPerFile(self):
181	"""
182	list the event collections structure by file
183	"""
184	return self.eventsPerFile
185
186	# #################################################
187	def getFiles(self):
188	"""
189	return files grouped by fileblock
190	"""
191	return self.blocksinfo
192
193	########################################################################