CRAB/python/DataDiscovery.py

#!/usr/bin/env python
import sys, os, string, re
from DBSInfo import *


# ####################################
class DataDiscoveryError(exceptions.Exception):
    def __init__(self, errorMessage):
        args=errorMessage
        exceptions.Exception.__init__(self, args)
        pass

    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
class NotExistingDatasetError(exceptions.Exception):
    def __init__(self, errorMessage):
        args=errorMessage
        exceptions.Exception.__init__(self, args)
        pass

    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
class NoDataTierinProvenanceError(exceptions.Exception):
    def __init__(self, errorMessage):
        args=errorMessage
        exceptions.Exception.__init__(self, args)
        pass

    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
# class to find and extact info from published data
class DataDiscovery:
    def __init__(self, datasetPath, dataTiers, cfg_params):

#       Attributes
        self.datasetPath = datasetPath
        self.dataTiers = dataTiers
        self.cfg_params = cfg_params

        self.eventsPerBlock = {}  # DBS output: map fileblocks-events for collection
        self.eventsPerFile = {}   # DBS output: map files-events
        self.blocksinfo = {}  # DBS output: map fileblocks-files 
#DBS output: max events computed by method getMaxEvents 

# ####################################
    def fetchDBSInfo(self):
        """
        Contact DBS
        """

        ## get DBS URL
        try:
            dbs_url=self.cfg_params['CMSSW.dbs_url']
        except KeyError:
            dbs_url="http://cmsdoc.cern.ch/cms/test/aprom/DBS/CGIServer/prodquery"

        ## get info about the requested dataset
        try:
            dbs_instance=self.cfg_params['CMSSW.dbs_instance']
        except KeyError:
            dbs_instance="MCGlobal/Writer"
 
        dbs = DBSInfo(dbs_url, dbs_instance)
        try:
            self.datasets = dbs.getMatchingDatasets(self.datasetPath)
        except dbsCgiApi.DbsCgiExecutionError, msg:
            raise DataDiscoveryError(msg)

        if len(self.datasets) == 0:
            raise DataDiscoveryError("DatasetPath=%s unknown to DBS" %self.datasetPath)
        if len(self.datasets) > 1:
            raise DataDiscoveryError("DatasetPath=%s is ambiguous" %self.datasetPath)

        try:
            self.dbsdataset = self.datasets[0].get('datasetPathName')

            self.eventsPerBlock = dbs.getEventsPerBlock(self.dbsdataset)
            self.blocksinfo = dbs.getDatasetFileBlocks(self.dbsdataset)
            self.eventsPerFile = dbs.getEventsPerFile(self.dbsdataset)
        except DBSError, ex:
            raise DataDiscoveryError(ex.getErrorMessage())
        
        if len(self.eventsPerBlock) <= 0:
            raise NotExistingDatasetError (("\nNo data for %s in DBS\nPlease check"
                                            + " dataset path variables in crab.cfg")
                                            % self.dbsdataset)


# #################################################
    def getMaxEvents(self):
        """
        max events 
        """
        ## loop over the event collections 
        nevts=0       
        for evc_evts in self.eventsPerBlock.values():
            nevts=nevts+evc_evts

        return nevts

# #################################################
    def getEventsPerBlock(self):
        """
        list the event collections structure by fileblock 
        """
        return self.eventsPerBlock

# #################################################
    def getEventsPerFile(self):
        """
        list the event collections structure by file 
        """
        return self.eventsPerFile

# #################################################
    def getFiles(self):
        """
        return files grouped by fileblock 
        """
        return self.blocksinfo        

########################################################################
Revision:	1.12
Committed:	Fri Sep 22 14:49:17 2006 UTC (18 years, 7 months ago) by gutsche
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_1_5_0_pre2, CRAB_1_4_2, CRAB_1_5_0_pre1, CRAB_1_4_1, CRAB_1_4_1_pre2, CRAB_1_4_1_pre1, CRAB_1_4_0, CRAB_1_4_0_pre4, CRAB_1_4_0_pre3, CRAB_1_4_0_pre2, CRAB_1_4_0_pre1, CRAB_1_3_0, CRAB_1_3_0_pre6, CRAB_1_3_0_pre5, CRAB_1_3_0_pre4
Branch point for:	branch_1_4_1
Changes since 1.11:	+1 -6 lines
Log Message:	changed DBS default to MCGlobal/Writer
#	User	Rev	Content
1	gutsche	1.6	#!/usr/bin/env python
2	afanfani	1.1	import sys, os, string, re
3			from DBSInfo import *
4
5	afanfani	1.3
6	afanfani	1.1	# ####################################
7	afanfani	1.3	class DataDiscoveryError(exceptions.Exception):
8	slacapra	1.7	def __init__(self, errorMessage):
9			args=errorMessage
10			exceptions.Exception.__init__(self, args)
11			pass
12
13			def getErrorMessage(self):
14			""" Return exception error """
15			return "%s" % (self.args)
16	afanfani	1.3
17	afanfani	1.1	# ####################################
18	afanfani	1.3	class NotExistingDatasetError(exceptions.Exception):
19	slacapra	1.7	def __init__(self, errorMessage):
20			args=errorMessage
21			exceptions.Exception.__init__(self, args)
22			pass
23
24			def getErrorMessage(self):
25			""" Return exception error """
26			return "%s" % (self.args)
27	afanfani	1.1
28			# ####################################
29	afanfani	1.3	class NoDataTierinProvenanceError(exceptions.Exception):
30	slacapra	1.7	def __init__(self, errorMessage):
31			args=errorMessage
32			exceptions.Exception.__init__(self, args)
33			pass
34
35			def getErrorMessage(self):
36			""" Return exception error """
37			return "%s" % (self.args)
38	afanfani	1.1
39			# ####################################
40			# class to find and extact info from published data
41			class DataDiscovery:
42	slacapra	1.11	def __init__(self, datasetPath, dataTiers, cfg_params):
43	afanfani	1.1
44			# Attributes
45	slacapra	1.11	self.datasetPath = datasetPath
46	afanfani	1.1	self.dataTiers = dataTiers
47			self.cfg_params = cfg_params
48
49	slacapra	1.11	self.eventsPerBlock = {} # DBS output: map fileblocks-events for collection
50			self.eventsPerFile = {} # DBS output: map files-events
51			self.blocksinfo = {} # DBS output: map fileblocks-files
52	afanfani	1.1	#DBS output: max events computed by method getMaxEvents
53
54			# ####################################
55			def fetchDBSInfo(self):
56			"""
57			Contact DBS
58			"""
59
60	slacapra	1.11	## get DBS URL
61			try:
62			dbs_url=self.cfg_params['CMSSW.dbs_url']
63			except KeyError:
64			dbs_url="http://cmsdoc.cern.ch/cms/test/aprom/DBS/CGIServer/prodquery"
65	afanfani	1.3
66	slacapra	1.11	## get info about the requested dataset
67			try:
68			dbs_instance=self.cfg_params['CMSSW.dbs_instance']
69			except KeyError:
70	gutsche	1.12	dbs_instance="MCGlobal/Writer"
71	slacapra	1.11
72			dbs = DBSInfo(dbs_url, dbs_instance)
73	afanfani	1.5	try:
74	slacapra	1.11	self.datasets = dbs.getMatchingDatasets(self.datasetPath)
75			except dbsCgiApi.DbsCgiExecutionError, msg:
76			raise DataDiscoveryError(msg)
77
78	afanfani	1.4	if len(self.datasets) == 0:
79	slacapra	1.11	raise DataDiscoveryError("DatasetPath=%s unknown to DBS" %self.datasetPath)
80	afanfani	1.4	if len(self.datasets) > 1:
81	slacapra	1.11	raise DataDiscoveryError("DatasetPath=%s is ambiguous" %self.datasetPath)
82
83	afanfani	1.1	try:
84	slacapra	1.7	self.dbsdataset = self.datasets[0].get('datasetPathName')
85	slacapra	1.11
86			self.eventsPerBlock = dbs.getEventsPerBlock(self.dbsdataset)
87			self.blocksinfo = dbs.getDatasetFileBlocks(self.dbsdataset)
88			self.eventsPerFile = dbs.getEventsPerFile(self.dbsdataset)
89	afanfani	1.3	except DBSError, ex:
90	slacapra	1.7	raise DataDiscoveryError(ex.getErrorMessage())
91	afanfani	1.3
92	slacapra	1.11	if len(self.eventsPerBlock) <= 0:
93			raise NotExistingDatasetError (("\nNo data for %s in DBS\nPlease check"
94			+ " dataset path variables in crab.cfg")
95			% self.dbsdataset)
96	afanfani	1.1
97
98			# #################################################
99			def getMaxEvents(self):
100			"""
101	slacapra	1.11	max events
102	afanfani	1.1	"""
103	slacapra	1.11	## loop over the event collections
104	afanfani	1.1	nevts=0
105	slacapra	1.11	for evc_evts in self.eventsPerBlock.values():
106			nevts=nevts+evc_evts
107	afanfani	1.1
108			return nevts
109
110			# #################################################
111	slacapra	1.11	def getEventsPerBlock(self):
112	afanfani	1.1	"""
113	slacapra	1.11	list the event collections structure by fileblock
114	afanfani	1.1	"""
115	slacapra	1.11	return self.eventsPerBlock
116	afanfani	1.1
117			# #################################################
118	slacapra	1.11	def getEventsPerFile(self):
119	afanfani	1.1	"""
120	slacapra	1.11	list the event collections structure by file
121	afanfani	1.1	"""
122	slacapra	1.11	return self.eventsPerFile
123	afanfani	1.1
124			# #################################################
125	slacapra	1.11	def getFiles(self):
126	afanfani	1.1	"""
127	slacapra	1.11	return files grouped by fileblock
128	afanfani	1.1	"""
129	slacapra	1.11	return self.blocksinfo
130	afanfani	1.1
131			########################################################################