CRAB/python/DataDiscovery.py

#!/usr/bin/env python
import sys, os, string, re
from DBSInfo import *


# ####################################
class DataDiscoveryError(exceptions.Exception):
    def __init__(self, errorMessage):
        args=errorMessage
        exceptions.Exception.__init__(self, args)
        pass

    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
class NotExistingDatasetError(exceptions.Exception):
    def __init__(self, errorMessage):
        args=errorMessage
        exceptions.Exception.__init__(self, args)
        pass

    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
class NoDataTierinProvenanceError(exceptions.Exception):
    def __init__(self, errorMessage):
        args=errorMessage
        exceptions.Exception.__init__(self, args)
        pass

    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
# class to find and extact info from published data
class DataDiscovery:
    def __init__(self, datasetPath, dataTiers, cfg_params):

#       Attributes
        self.datasetPath = datasetPath
        self.dataTiers = dataTiers
        self.cfg_params = cfg_params

        self.eventsPerBlock = {}  # DBS output: map fileblocks-events for collection
        self.eventsPerFile = {}   # DBS output: map files-events
        self.blocksinfo = {}  # DBS output: map fileblocks-files 
#DBS output: max events computed by method getMaxEvents 

# ####################################
    def fetchDBSInfo(self):
        """
        Contact DBS
        """

        ## get DBS URL
        try:
            dbs_url=self.cfg_params['CMSSW.dbs_url']
        except KeyError:
            # This URL is deprecated and no longer supported by the DBS team
            #dbs_url="http://cmsdoc.cern.ch/cms/aprom/DBS/CGIServer/prodquery"
            # This URL is supported by the DBS team and is 'stable'
            dbs_url="http://cmsdoc.cern.ch/cms/test/aprom/DBS/CGIServer/prodquery"
            # This URL is used for testing of the most recent changes by the DBS team
            #dbs_url = "http://cmsdoc.cern.ch/cms/test/aprom/DBS/CGIServer/prodquerytest"

        ## get info about the requested dataset
        try:
            dbs_instance=self.cfg_params['CMSSW.dbs_instance']
        except KeyError:
            dbs_instance="MCLocal/Writer"
 
        dbs = DBSInfo(dbs_url, dbs_instance)
        try:
            self.datasets = dbs.getMatchingDatasets(self.datasetPath)
        except dbsCgiApi.DbsCgiExecutionError, msg:
            raise DataDiscoveryError(msg)

        if len(self.datasets) == 0:
            raise DataDiscoveryError("DatasetPath=%s unknown to DBS" %self.datasetPath)
        if len(self.datasets) > 1:
            raise DataDiscoveryError("DatasetPath=%s is ambiguous" %self.datasetPath)

        try:
            self.dbsdataset = self.datasets[0].get('datasetPathName')

            self.eventsPerBlock = dbs.getEventsPerBlock(self.dbsdataset)
            self.blocksinfo = dbs.getDatasetFileBlocks(self.dbsdataset)
            self.eventsPerFile = dbs.getEventsPerFile(self.dbsdataset)
        except DBSError, ex:
            raise DataDiscoveryError(ex.getErrorMessage())
        
        if len(self.eventsPerBlock) <= 0:
            raise NotExistingDatasetError (("\nNo data for %s in DBS\nPlease check"
                                            + " dataset path variables in crab.cfg")
                                            % self.dbsdataset)


# #################################################
    def getMaxEvents(self):
        """
        max events 
        """
        ## loop over the event collections 
        nevts=0       
        for evc_evts in self.eventsPerBlock.values():
            nevts=nevts+evc_evts

        return nevts

# #################################################
    def getEventsPerBlock(self):
        """
        list the event collections structure by fileblock 
        """
        return self.eventsPerBlock

# #################################################
    def getEventsPerFile(self):
        """
        list the event collections structure by file 
        """
        return self.eventsPerFile

# #################################################
    def getFiles(self):
        """
        return files grouped by fileblock 
        """
        return self.blocksinfo        

########################################################################
Revision:	1.11
Committed:	Wed Sep 20 17:29:52 2006 UTC (18 years, 7 months ago) by slacapra
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_1_3_0_pre3
Changes since 1.10:	+50 -73 lines
Log Message:	BOSS4 + sub-file splitting + taskDB
#	User	Rev	Content
1	gutsche	1.6	#!/usr/bin/env python
2	afanfani	1.1	import sys, os, string, re
3			from DBSInfo import *
4
5	afanfani	1.3
6	afanfani	1.1	# ####################################
7	afanfani	1.3	class DataDiscoveryError(exceptions.Exception):
8	slacapra	1.7	def __init__(self, errorMessage):
9			args=errorMessage
10			exceptions.Exception.__init__(self, args)
11			pass
12
13			def getErrorMessage(self):
14			""" Return exception error """
15			return "%s" % (self.args)
16	afanfani	1.3
17	afanfani	1.1	# ####################################
18	afanfani	1.3	class NotExistingDatasetError(exceptions.Exception):
19	slacapra	1.7	def __init__(self, errorMessage):
20			args=errorMessage
21			exceptions.Exception.__init__(self, args)
22			pass
23
24			def getErrorMessage(self):
25			""" Return exception error """
26			return "%s" % (self.args)
27	afanfani	1.1
28			# ####################################
29	afanfani	1.3	class NoDataTierinProvenanceError(exceptions.Exception):
30	slacapra	1.7	def __init__(self, errorMessage):
31			args=errorMessage
32			exceptions.Exception.__init__(self, args)
33			pass
34
35			def getErrorMessage(self):
36			""" Return exception error """
37			return "%s" % (self.args)
38	afanfani	1.1
39			# ####################################
40			# class to find and extact info from published data
41			class DataDiscovery:
42	slacapra	1.11	def __init__(self, datasetPath, dataTiers, cfg_params):
43	afanfani	1.1
44			# Attributes
45	slacapra	1.11	self.datasetPath = datasetPath
46	afanfani	1.1	self.dataTiers = dataTiers
47			self.cfg_params = cfg_params
48
49	slacapra	1.11	self.eventsPerBlock = {} # DBS output: map fileblocks-events for collection
50			self.eventsPerFile = {} # DBS output: map files-events
51			self.blocksinfo = {} # DBS output: map fileblocks-files
52	afanfani	1.1	#DBS output: max events computed by method getMaxEvents
53
54			# ####################################
55			def fetchDBSInfo(self):
56			"""
57			Contact DBS
58			"""
59
60	slacapra	1.11	## get DBS URL
61			try:
62			dbs_url=self.cfg_params['CMSSW.dbs_url']
63			except KeyError:
64			# This URL is deprecated and no longer supported by the DBS team
65			#dbs_url="http://cmsdoc.cern.ch/cms/aprom/DBS/CGIServer/prodquery"
66			# This URL is supported by the DBS team and is 'stable'
67			dbs_url="http://cmsdoc.cern.ch/cms/test/aprom/DBS/CGIServer/prodquery"
68			# This URL is used for testing of the most recent changes by the DBS team
69			#dbs_url = "http://cmsdoc.cern.ch/cms/test/aprom/DBS/CGIServer/prodquerytest"
70	afanfani	1.3
71	slacapra	1.11	## get info about the requested dataset
72			try:
73			dbs_instance=self.cfg_params['CMSSW.dbs_instance']
74			except KeyError:
75			dbs_instance="MCLocal/Writer"
76
77			dbs = DBSInfo(dbs_url, dbs_instance)
78	afanfani	1.5	try:
79	slacapra	1.11	self.datasets = dbs.getMatchingDatasets(self.datasetPath)
80			except dbsCgiApi.DbsCgiExecutionError, msg:
81			raise DataDiscoveryError(msg)
82
83	afanfani	1.4	if len(self.datasets) == 0:
84	slacapra	1.11	raise DataDiscoveryError("DatasetPath=%s unknown to DBS" %self.datasetPath)
85	afanfani	1.4	if len(self.datasets) > 1:
86	slacapra	1.11	raise DataDiscoveryError("DatasetPath=%s is ambiguous" %self.datasetPath)
87
88	afanfani	1.1	try:
89	slacapra	1.7	self.dbsdataset = self.datasets[0].get('datasetPathName')
90	slacapra	1.11
91			self.eventsPerBlock = dbs.getEventsPerBlock(self.dbsdataset)
92			self.blocksinfo = dbs.getDatasetFileBlocks(self.dbsdataset)
93			self.eventsPerFile = dbs.getEventsPerFile(self.dbsdataset)
94	afanfani	1.3	except DBSError, ex:
95	slacapra	1.7	raise DataDiscoveryError(ex.getErrorMessage())
96	afanfani	1.3
97	slacapra	1.11	if len(self.eventsPerBlock) <= 0:
98			raise NotExistingDatasetError (("\nNo data for %s in DBS\nPlease check"
99			+ " dataset path variables in crab.cfg")
100			% self.dbsdataset)
101	afanfani	1.1
102
103			# #################################################
104			def getMaxEvents(self):
105			"""
106	slacapra	1.11	max events
107	afanfani	1.1	"""
108	slacapra	1.11	## loop over the event collections
109	afanfani	1.1	nevts=0
110	slacapra	1.11	for evc_evts in self.eventsPerBlock.values():
111			nevts=nevts+evc_evts
112	afanfani	1.1
113			return nevts
114
115			# #################################################
116	slacapra	1.11	def getEventsPerBlock(self):
117	afanfani	1.1	"""
118	slacapra	1.11	list the event collections structure by fileblock
119	afanfani	1.1	"""
120	slacapra	1.11	return self.eventsPerBlock
121	afanfani	1.1
122			# #################################################
123	slacapra	1.11	def getEventsPerFile(self):
124	afanfani	1.1	"""
125	slacapra	1.11	list the event collections structure by file
126	afanfani	1.1	"""
127	slacapra	1.11	return self.eventsPerFile
128	afanfani	1.1
129			# #################################################
130	slacapra	1.11	def getFiles(self):
131	afanfani	1.1	"""
132	slacapra	1.11	return files grouped by fileblock
133	afanfani	1.1	"""
134	slacapra	1.11	return self.blocksinfo
135	afanfani	1.1
136			########################################################################