CRAB/python/DataDiscovery.py

#!/usr/bin/env python
import exceptions
import DBSAPI.dbsApi
from DBSAPI.dbsApiException import * 
import common
from crab_util import *
import os 


# #######################################
class DBSError(exceptions.Exception):
    def __init__(self, errorName, errorMessage):
        args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
        exceptions.Exception.__init__(self, args)
        pass
    
    def getErrorMessage(self):
        """ Return error message """
        return "%s" % (self.args)

# #######################################
class DBSInvalidDataTierError(exceptions.Exception):
    def __init__(self, errorName, errorMessage):
        args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
        exceptions.Exception.__init__(self, args)
        pass
    
    def getErrorMessage(self):
        """ Return error message """
        return "%s" % (self.args)

# #######################################
class DBSInfoError:
    def __init__(self, url):
        print '\nERROR accessing DBS url : '+url+'\n'
        pass

# ####################################
class DataDiscoveryError(exceptions.Exception):
    def __init__(self, errorMessage):
        self.args=errorMessage
        exceptions.Exception.__init__(self, self.args)
        pass

    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
class NotExistingDatasetError(exceptions.Exception):
    def __init__(self, errorMessage):
        self.args=errorMessage
        exceptions.Exception.__init__(self, self.args)
        pass

    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
class NoDataTierinProvenanceError(exceptions.Exception):
    def __init__(self, errorMessage):
        self.args=errorMessage
        exceptions.Exception.__init__(self, self.args)
        pass

    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
# class to find and extact info from published data
class DataDiscovery:
    def __init__(self, datasetPath, cfg_params, skipAnBlocks):

        #       Attributes
        self.datasetPath = datasetPath
        self.cfg_params = cfg_params
        self.skipBlocks = skipAnBlocks

        self.eventsPerBlock = {}  # DBS output: map fileblocks-events for collection
        self.eventsPerFile = {}   # DBS output: map files-events
        self.blocksinfo = {}      # DBS output: map fileblocks-files 
        self.maxEvents = 0        # DBS output: max events
        self.parent = {}       # DBS output: max events

# ####################################
    def fetchDBSInfo(self):
        """
        Contact DBS
        """
        ## get DBS URL
        global_url="http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
        caf_url = "http://cmsdbsprod.cern.ch/cms_dbs_caf_analysis_01/servlet/DBSServlet"
        dbs_url_map  =   {'glite':    global_url,
                          'glitecoll':global_url,\
                          'condor':   global_url,\
                          'condor_g': global_url,\
                          'glidein':  global_url,\
                          'lsf':      global_url,\
                          'caf':      caf_url,\
                          'sge':      global_url,
                          'arc':      global_url
                          }

        dbs_url_default = dbs_url_map[(common.scheduler.name()).lower()]
        dbs_url=  self.cfg_params.get('CMSSW.dbs_url', dbs_url_default)
        common.logger.debug(3,"Accessing DBS at: "+dbs_url)

        ## check if runs are selected
        runselection = []
        if (self.cfg_params.has_key('CMSSW.runselection')):
            runselection = parseRange2(self.cfg_params['CMSSW.runselection'])


        self.splitByRun = int(self.cfg_params.get('CMSSW.split_by_run', 0))
          
        self.ads = int(self.cfg_params.get('CMSSW.ads', 0))

        common.logger.debug(6,"runselection is: %s"%runselection)
        ## service API
        args = {}
        args['url']     = dbs_url
        args['level']   = 'CRITICAL'

        ## check if has been requested to use the parent info
        useparent = int(self.cfg_params.get('CMSSW.use_parent',0))

        ## check if has been asked for a non default file to store/read analyzed fileBlocks   
        defaultName = common.work_space.shareDir()+'AnalyzedBlocks.txt'  
        fileBlocks_FileName = os.path.abspath(self.cfg_params.get('CMSSW.fileblocks_file',defaultName))
 
        api = DBSAPI.dbsApi.DbsApi(args)

        self.files = self.queryDbs(api,path=self.datasetPath,runselection=runselection,useParent=useparent)

        anFileBlocks = []
        if self.skipBlocks: anFileBlocks = readTXTfile(self, fileBlocks_FileName) 

        # parse files and fill arrays
        for file in self.files :
            parList = []
            # skip already analyzed blocks
            fileblock = file['Block']['Name']
            if fileblock not in anFileBlocks :
                filename = file['LogicalFileName']
                # asked retry the list of parent for the given child 
                if useparent==1: parList = [x['LogicalFileName'] for x in file['ParentList']] 
                self.parent[filename] = parList 
                if filename.find('.dat') < 0 :
                    events    = file['NumberOfEvents']
                    # number of events per block
                    if fileblock in self.eventsPerBlock.keys() :
                        self.eventsPerBlock[fileblock] += events
                    else :
                        self.eventsPerBlock[fileblock] = events
                    # number of events per file
                    self.eventsPerFile[filename] = events
            
                    # number of events per block
                    if fileblock in self.blocksinfo.keys() :
                        self.blocksinfo[fileblock].append(filename)
                    else :
                        self.blocksinfo[fileblock] = [filename]
            
                    # total number of events
                    self.maxEvents += events
        if  self.skipBlocks and len(self.eventsPerBlock.keys()) == 0:
            msg = "No new fileblocks available for dataset: "+str(self.datasetPath)
            raise  CrabException(msg)    

        saveFblocks='' 
        for block in self.eventsPerBlock.keys() :
            saveFblocks += str(block)+'\n' 
            common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(self.eventsPerBlock[block],block))
        writeTXTfile(self, fileBlocks_FileName , saveFblocks) 
                      
        if len(self.eventsPerBlock) <= 0:
            raise NotExistingDatasetError(("\nNo data for %s in DBS\nPlease check"
                                            + " dataset path variables in crab.cfg")
                                            % self.datasetPath)


###########################

    def queryDbs(self,api,path=None,runselection=None,useParent=None):
 
        allowedRetriveValue = ['retrive_block', 'retrive_run']
        if useParent==1 : allowedRetriveValue = allowedRetriveValue + ['retrive_parent']
        try:
            if len(runselection) <=0 :
                if useParent==1 or self.splitByRun==1 :
                    if self.ads==1 :           
                        files = api.listFiles(analysisDataset=path, retriveList=allowedRetriveValue) 
                    else :
                        files = api.listFiles(path=path, retriveList=allowedRetriveValue) 
                    common.logger.debug(5,"Set of input parameters used for DBS query : \n"+str(allowedRetriveValue)) 
                    common.logger.write("Set of input parameters used for DBS query : \n"+str(allowedRetriveValue)) 
                else:
                    files = api.listDatasetFiles(self.datasetPath)
            else :
                files=[]
                for arun in runselection:
                    try:
                        if self.ads==1 : filesinrun = api.listFiles(analysisDataset=path,retriveList=allowedRetriveValue,runNumber=arun)
                        else: filesinrun = api.listFiles(path=path,retriveList=allowedRetriveValue,runNumber=arun)
                        files.extend(filesinrun)
                    except:
                        msg="WARNING: problem extracting info from DBS for run %s "%arun
                        common.logger.message(msg)
                        pass

        except DbsBadRequest, msg:
            raise DataDiscoveryError(msg)
        except DBSError, msg:
            raise DataDiscoveryError(msg)

        return files

# #################################################
    def getMaxEvents(self):
        """
        max events 
        """
        return self.maxEvents

# #################################################
    def getEventsPerBlock(self):
        """
        list the event collections structure by fileblock 
        """
        return self.eventsPerBlock

# #################################################
    def getEventsPerFile(self):
        """
        list the event collections structure by file 
        """
        return self.eventsPerFile

# #################################################
    def getFiles(self):
        """
        return files grouped by fileblock 
        """
        return self.blocksinfo        

# #################################################
    def getParent(self):
        """
        return parent grouped by file 
        """
        return self.parent        

# #################################################
    def getListFiles(self):
        """
        return parent grouped by file 
        """
        return self.files        

########################################################################
Revision:	1.30
Committed:	Tue Mar 31 22:36:42 2009 UTC (16 years ago) by spiga
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_2_6_0_pre2, CRAB_2_6_0_pre1, CRAB_2_5_1, CRAB_2_5_1_pre4, CRAB_2_5_1_pre3, CRAB_2_5_1_pre2, CRAB_2_5_1_pre1
Changes since 1.29:	+2 -5 lines
Log Message:	fix for use_parent bug introduced into 2_5_0
#	User	Rev	Content
1	gutsche	1.6	#!/usr/bin/env python
2	slacapra	1.18	import exceptions
3			import DBSAPI.dbsApi
4			from DBSAPI.dbsApiException import *
5			import common
6			from crab_util import *
7	spiga	1.22	import os
8	afanfani	1.1
9	afanfani	1.3
10	slacapra	1.18	# #######################################
11			class DBSError(exceptions.Exception):
12			def __init__(self, errorName, errorMessage):
13			args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
14			exceptions.Exception.__init__(self, args)
15			pass
16
17			def getErrorMessage(self):
18			""" Return error message """
19			return "%s" % (self.args)
20
21			# #######################################
22			class DBSInvalidDataTierError(exceptions.Exception):
23			def __init__(self, errorName, errorMessage):
24			args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
25			exceptions.Exception.__init__(self, args)
26			pass
27
28			def getErrorMessage(self):
29			""" Return error message """
30			return "%s" % (self.args)
31
32			# #######################################
33			class DBSInfoError:
34			def __init__(self, url):
35			print '\nERROR accessing DBS url : '+url+'\n'
36			pass
37
38	afanfani	1.1	# ####################################
39	afanfani	1.3	class DataDiscoveryError(exceptions.Exception):
40	slacapra	1.7	def __init__(self, errorMessage):
41	gutsche	1.15	self.args=errorMessage
42	slacapra	1.14	exceptions.Exception.__init__(self, self.args)
43	slacapra	1.7	pass
44
45			def getErrorMessage(self):
46			""" Return exception error """
47			return "%s" % (self.args)
48	afanfani	1.3
49	afanfani	1.1	# ####################################
50	afanfani	1.3	class NotExistingDatasetError(exceptions.Exception):
51	slacapra	1.7	def __init__(self, errorMessage):
52	gutsche	1.15	self.args=errorMessage
53	slacapra	1.14	exceptions.Exception.__init__(self, self.args)
54	slacapra	1.7	pass
55
56			def getErrorMessage(self):
57			""" Return exception error """
58			return "%s" % (self.args)
59	afanfani	1.1
60			# ####################################
61	afanfani	1.3	class NoDataTierinProvenanceError(exceptions.Exception):
62	slacapra	1.7	def __init__(self, errorMessage):
63	gutsche	1.15	self.args=errorMessage
64	slacapra	1.14	exceptions.Exception.__init__(self, self.args)
65	slacapra	1.7	pass
66
67			def getErrorMessage(self):
68			""" Return exception error """
69			return "%s" % (self.args)
70	afanfani	1.1
71			# ####################################
72			# class to find and extact info from published data
73			class DataDiscovery:
74	spiga	1.22	def __init__(self, datasetPath, cfg_params, skipAnBlocks):
75	afanfani	1.1
76	slacapra	1.18	# Attributes
77	slacapra	1.11	self.datasetPath = datasetPath
78	afanfani	1.1	self.cfg_params = cfg_params
79	spiga	1.22	self.skipBlocks = skipAnBlocks
80	afanfani	1.1
81	slacapra	1.11	self.eventsPerBlock = {} # DBS output: map fileblocks-events for collection
82			self.eventsPerFile = {} # DBS output: map files-events
83	slacapra	1.18	self.blocksinfo = {} # DBS output: map fileblocks-files
84			self.maxEvents = 0 # DBS output: max events
85	spiga	1.21	self.parent = {} # DBS output: max events
86	afanfani	1.1
87			# ####################################
88			def fetchDBSInfo(self):
89			"""
90			Contact DBS
91			"""
92	slacapra	1.11	## get DBS URL
93	spiga	1.25	global_url="http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
94			caf_url = "http://cmsdbsprod.cern.ch/cms_dbs_caf_analysis_01/servlet/DBSServlet"
95			dbs_url_map = {'glite': global_url,
96			'glitecoll':global_url,\
97			'condor': global_url,\
98			'condor_g': global_url,\
99			'glidein': global_url,\
100			'lsf': global_url,\
101			'caf': caf_url,\
102	edelmann	1.29	'sge': global_url,
103			'arc': global_url
104	spiga	1.25	}
105	afanfani	1.3
106	spiga	1.25	dbs_url_default = dbs_url_map[(common.scheduler.name()).lower()]
107			dbs_url= self.cfg_params.get('CMSSW.dbs_url', dbs_url_default)
108	slacapra	1.18	common.logger.debug(3,"Accessing DBS at: "+dbs_url)
109
110			## check if runs are selected
111	slacapra	1.19	runselection = []
112			if (self.cfg_params.has_key('CMSSW.runselection')):
113	slacapra	1.18	runselection = parseRange2(self.cfg_params['CMSSW.runselection'])
114
115	spiga	1.26
116			self.splitByRun = int(self.cfg_params.get('CMSSW.split_by_run', 0))
117
118			self.ads = int(self.cfg_params.get('CMSSW.ads', 0))
119
120	afanfani	1.20	common.logger.debug(6,"runselection is: %s"%runselection)
121	slacapra	1.18	## service API
122			args = {}
123			args['url'] = dbs_url
124			args['level'] = 'CRITICAL'
125
126	spiga	1.21	## check if has been requested to use the parent info
127	spiga	1.26	useparent = int(self.cfg_params.get('CMSSW.use_parent',0))
128	spiga	1.21
129	spiga	1.22	## check if has been asked for a non default file to store/read analyzed fileBlocks
130			defaultName = common.work_space.shareDir()+'AnalyzedBlocks.txt'
131			fileBlocks_FileName = os.path.abspath(self.cfg_params.get('CMSSW.fileblocks_file',defaultName))
132
133	slacapra	1.18	api = DBSAPI.dbsApi.DbsApi(args)
134	afanfani	1.20
135	spiga	1.27	self.files = self.queryDbs(api,path=self.datasetPath,runselection=runselection,useParent=useparent)
136	slacapra	1.11
137	spiga	1.22	anFileBlocks = []
138			if self.skipBlocks: anFileBlocks = readTXTfile(self, fileBlocks_FileName)
139
140	slacapra	1.18	# parse files and fill arrays
141	spiga	1.26	for file in self.files :
142	spiga	1.21	parList = []
143	spiga	1.22	# skip already analyzed blocks
144			fileblock = file['Block']['Name']
145			if fileblock not in anFileBlocks :
146			filename = file['LogicalFileName']
147			# asked retry the list of parent for the given child
148	spiga	1.26	if useparent==1: parList = [x['LogicalFileName'] for x in file['ParentList']]
149	spiga	1.22	self.parent[filename] = parList
150			if filename.find('.dat') < 0 :
151			events = file['NumberOfEvents']
152			# number of events per block
153			if fileblock in self.eventsPerBlock.keys() :
154			self.eventsPerBlock[fileblock] += events
155			else :
156			self.eventsPerBlock[fileblock] = events
157			# number of events per file
158			self.eventsPerFile[filename] = events
159
160			# number of events per block
161			if fileblock in self.blocksinfo.keys() :
162			self.blocksinfo[fileblock].append(filename)
163			else :
164			self.blocksinfo[fileblock] = [filename]
165
166			# total number of events
167			self.maxEvents += events
168			if self.skipBlocks and len(self.eventsPerBlock.keys()) == 0:
169			msg = "No new fileblocks available for dataset: "+str(self.datasetPath)
170			raise CrabException(msg)
171	slacapra	1.11
172	spiga	1.22	saveFblocks=''
173	slacapra	1.18	for block in self.eventsPerBlock.keys() :
174	spiga	1.22	saveFblocks += str(block)+'\n'
175	slacapra	1.18	common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(self.eventsPerBlock[block],block))
176	spiga	1.22	writeTXTfile(self, fileBlocks_FileName , saveFblocks)
177
178	slacapra	1.11	if len(self.eventsPerBlock) <= 0:
179	slacapra	1.18	raise NotExistingDatasetError(("\nNo data for %s in DBS\nPlease check"
180	slacapra	1.11	+ " dataset path variables in crab.cfg")
181	slacapra	1.18	% self.datasetPath)
182	afanfani	1.1
183
184	spiga	1.26	###########################
185
186	spiga	1.27	def queryDbs(self,api,path=None,runselection=None,useParent=None):
187	spiga	1.26
188	spiga	1.30	allowedRetriveValue = ['retrive_block', 'retrive_run']
189			if useParent==1 : allowedRetriveValue = allowedRetriveValue + ['retrive_parent']
190	spiga	1.26	try:
191	spiga	1.27	if len(runselection) <=0 :
192	spiga	1.26	if useParent==1 or self.splitByRun==1 :
193			if self.ads==1 :
194			files = api.listFiles(analysisDataset=path, retriveList=allowedRetriveValue)
195			else :
196			files = api.listFiles(path=path, retriveList=allowedRetriveValue)
197			common.logger.debug(5,"Set of input parameters used for DBS query : \n"+str(allowedRetriveValue))
198			common.logger.write("Set of input parameters used for DBS query : \n"+str(allowedRetriveValue))
199			else:
200			files = api.listDatasetFiles(self.datasetPath)
201			else :
202			files=[]
203			for arun in runselection:
204			try:
205			if self.ads==1 : filesinrun = api.listFiles(analysisDataset=path,retriveList=allowedRetriveValue,runNumber=arun)
206			else: filesinrun = api.listFiles(path=path,retriveList=allowedRetriveValue,runNumber=arun)
207			files.extend(filesinrun)
208			except:
209			msg="WARNING: problem extracting info from DBS for run %s "%arun
210			common.logger.message(msg)
211			pass
212
213			except DbsBadRequest, msg:
214			raise DataDiscoveryError(msg)
215			except DBSError, msg:
216			raise DataDiscoveryError(msg)
217
218			return files
219
220	afanfani	1.1	# #################################################
221			def getMaxEvents(self):
222			"""
223	slacapra	1.11	max events
224	afanfani	1.1	"""
225	slacapra	1.18	return self.maxEvents
226	afanfani	1.1
227			# #################################################
228	slacapra	1.11	def getEventsPerBlock(self):
229	afanfani	1.1	"""
230	slacapra	1.11	list the event collections structure by fileblock
231	afanfani	1.1	"""
232	slacapra	1.11	return self.eventsPerBlock
233	afanfani	1.1
234			# #################################################
235	slacapra	1.11	def getEventsPerFile(self):
236	afanfani	1.1	"""
237	slacapra	1.11	list the event collections structure by file
238	afanfani	1.1	"""
239	slacapra	1.11	return self.eventsPerFile
240	afanfani	1.1
241			# #################################################
242	slacapra	1.11	def getFiles(self):
243	afanfani	1.1	"""
244	slacapra	1.11	return files grouped by fileblock
245	afanfani	1.1	"""
246	slacapra	1.11	return self.blocksinfo
247	afanfani	1.1
248	spiga	1.21	# #################################################
249			def getParent(self):
250			"""
251			return parent grouped by file
252			"""
253			return self.parent
254
255	spiga	1.26	# #################################################
256			def getListFiles(self):
257			"""
258			return parent grouped by file
259			"""
260			return self.files
261
262	afanfani	1.1	########################################################################