ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/DBSInfo_EDM.py

Comparing COMP/CRAB/python/DBSInfo_EDM.py (file contents):
Revision 1.3 by afanfani, Tue May 30 22:54:49 2006 UTC vs.
Revision 1.6 by gutsche, Thu Aug 3 22:44:40 2006 UTC

#	Line 1 | Line 1
1		#!/usr/bin/env python
2		import sys, os, string, re, commands
3		import exceptions
4	<
4	>	import common
5	>	from crab_exceptions import *
6		try:
7		import dbsCgiApi
8		import dbsApi
#	Line 19 | Line 20 | except:
20		## for python 2.2 add the pyexpat.so to PYTHONPATH
21		pythonV=sys.version.split(' ')[0]
22		if pythonV.find('2.2') >= 0 :
23	<	Crabpydir=commands.getoutput('which crab')
24	<	Topdir=string.replace(Crabpydir,'/python/crab','')
25	<	extradir=Topdir+'/DLSAPI/extra'
26	<	if sys.path.count(extradir) <= 0:
27	<	if os.path.exists(extradir):
28	<	sys.path.insert(0, extradir)
23	>	Crabpydir=commands.getoutput('which crab')
24	>	Topdir=string.replace(Crabpydir,'/python/crab','')
25	>	extradir=Topdir+'/DLSAPI/extra'
26	>	if sys.path.count(extradir) <= 0:
27	>	if os.path.exists(extradir):
28	>	sys.path.insert(0, extradir)
29
30		# #######################################
31		class DBSError(exceptions.Exception):
32	<	def __init__(self, errorName, errorMessage):
33	<	args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
34	<	exceptions.Exception.__init__(self, args)
35	<	pass
36	<
37	<	def getErrorMessage(self):
38	<	""" Return error message """
39	<	return "%s" % (self.args)
32	>	def __init__(self, errorName, errorMessage):
33	>	args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
34	>	exceptions.Exception.__init__(self, args)
35	>	pass
36	>
37	>	def getErrorMessage(self):
38	>	""" Return error message """
39	>	return "%s" % (self.args)
40
41		# #######################################
42		class DBSInvalidDataTierError(exceptions.Exception):
43	<	def __init__(self, errorName, errorMessage):
44	<	args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
45	<	exceptions.Exception.__init__(self, args)
46	<	pass
47	<
48	<	def getErrorMessage(self):
49	<	""" Return error message """
50	<	return "%s" % (self.args)
43	>	def __init__(self, errorName, errorMessage):
44	>	args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
45	>	exceptions.Exception.__init__(self, args)
46	>	pass
47	>
48	>	def getErrorMessage(self):
49	>	""" Return error message """
50	>	return "%s" % (self.args)
51
52		# #######################################
53		class DBSInfoError:
54	<	def __init__(self, url):
55	<	print '\nERROR accessing DBS url : '+url+'\n'
56	<	pass
54	>	def __init__(self, url):
55	>	print '\nERROR accessing DBS url : '+url+'\n'
56	>	pass
57
58		##################################################################################
59		# Class to extract info from DBS
60		###############################################################################
61
62		class DBSInfo_EDM:
63	<	def __init__(self, dbs_instance):
64	<	"""
65	<	Construct api object.
66	<	"""
67	<	## cgi service API
68	<	DEFAULT_URL = "http://cmsdoc.cern.ch/cms/aprom/DBS/CGIServer/prodquery"
69	<	args = {}
70	<	args['instance']=dbs_instance
71	<
72	<	self.api = dbsCgiApi.DbsCgiApi(DEFAULT_URL, args)
73	<	## set log level
74	<	# self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_INFO_)
75	<	#self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_QUIET_)
76	<
77	<	def getMatchingDatasets (self, datasetPath):
78	<	""" Query DBS to get provenance """
79	<	try:
80	<	list = self.api.listProcessedDatasets("%s" %datasetPath)
81	<	except dbsApi.InvalidDataTier, ex:
82	<	raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
83	<	except dbsApi.DbsApiException, ex:
84	<	raise DBSError(ex.getClassName(),ex.getErrorMessage())
85	<	except dbsCgiApi.DbsCgiToolError , ex:
86	<	raise DBSError(ex.getClassName(),ex.getErrorMessage())
87	<
88	<	return list
89	<
90	<
91	<	def getDatasetProvenance(self, path, dataTiers):
92	<	""" Query DBS to get provenance """
93	<	try:
94	<	datasetParentList = self.api.getDatasetProvenance(path,dataTiers)
95	<	except dbsApi.InvalidDataTier, ex:
96	<	raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
97	<	except dbsApi.DbsApiException, ex:
98	<	raise DBSError(ex.getClassName(),ex.getErrorMessage())
99	<	return datasetParentList
100	<
101	<	def getDatasetContents(self, path):
102	<	""" Query DBS to get event collections """
103	<	# count events per block
104	<	nevtsbyblock = {}
105	<	try:
106	<	for fileBlock in self.api.getDatasetContents (path):
107	<	## get the event collections for each block
108	<	nevts = 0
109	<	for evc in fileBlock.get('eventCollectionList'):
110	<	nevts = nevts + evc.get('numberOfEvents')
111	<	print "DBSInfo: total nevts %i in block %s "%(nevts,fileBlock.get('blockName'))
112	<	nevtsbyblock[fileBlock.get('blockName')]=nevts
113	<	except dbsApi.DbsApiException, ex:
114	<	raise DBSError(ex.getClassName(),ex.getErrorMessage())
115	<
116	<	# returning a map of fileblock-nevts  will be enough for now
117	<	# TODO: in future the EvC collections grouped by fileblock should be returned
118	<	return nevtsbyblock
119	<
120	<
121	<	def getDatasetFileBlocks(self, path):
122	<	""" Query DBS to get files/fileblocks """
123	<	try:
124	<	FilesbyBlock={}
125	<	for fileBlock in self.api.getDatasetFileBlocks(path):
126	<	blockname=fileBlock.get('blockName')
127	<	filesinblock=[]
128	<	for files in fileBlock.get('fileList'):
129	<	#print "  block %s has file %s"%(blockname,files.getLogicalFileName())
130	<	filesinblock.append(files.get('logicalFileName'))
131	<	FilesbyBlock[blockname]=filesinblock
132	<	except dbsApi.DbsApiException, ex:
133	<	raise DBSError(ex.getClassName(),ex.getErrorMessage())
134	<
135	<	return FilesbyBlock
63	>	def __init__(self, dbs_url, dbs_instance):
64	>	"""
65	>	Construct api object.
66	>	"""
67	>	## cgi service API
68	>	args = {}
69	>	args['instance']=dbs_instance
70	>
71	>	common.logger.debug(3,"Accessing DBS at: "+dbs_url+" "+dbs_instance)
72	>
73	>	self.api = dbsCgiApi.DbsCgiApi(dbs_url, args)
74	>	## set log level
75	>	# self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_INFO_)
76	>	#self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_QUIET_)
77	>
78	>	def getMatchingDatasets (self, datasetPath):
79	>	""" Query DBS to get provenance """
80	>	try:
81	>	list = self.api.listProcessedDatasets("%s" %datasetPath)
82	>	except dbsApi.InvalidDataTier, ex:
83	>	raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
84	>	except dbsApi.DbsApiException, ex:
85	>	raise DBSError(ex.getClassName(),ex.getErrorMessage())
86	>	except dbsCgiApi.DbsCgiToolError , ex:
87	>	raise DBSError(ex.getClassName(),ex.getErrorMessage())
88	>	except dbsCgiApi.DbsCgiBadResponse , ex:
89	>	raise DBSError(ex.getClassName(),ex.getErrorMessage())
90	>
91	>	return list
92	>
93	>
94	>	def getDatasetProvenance(self, path, dataTiers):
95	>	""" Query DBS to get provenance """
96	>	try:
97	>	datasetParentList = self.api.getDatasetProvenance(path,dataTiers)
98	>	except dbsApi.InvalidDataTier, ex:
99	>	raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
100	>	except dbsApi.DbsApiException, ex:
101	>	raise DBSError(ex.getClassName(),ex.getErrorMessage())
102	>	return datasetParentList
103	>
104	>	def getDatasetContents(self, path):
105	>	""" Query DBS to get event collections """
106	>	# count events per block
107	>	nevtsbyblock = {}
108	>	#print "FileBlock :",str(self.api.getDatasetContents (path))
109	>	try:
110	>	for fileBlock in self.api.getDatasetContents (path):
111	>	## get the event collections for each block
112	>	nevts = 0
113	>	for evc in fileBlock.get('eventCollectionList'):
114	>	nevts = nevts + evc.get('numberOfEvents')
115	>	common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(nevts,fileBlock.get('blockName')))
116	>	#print "BlockName ",fileBlock.get('blockName')
117	>	## SL temp hack to get rid of a mismatch between block names as returned by DBS
118	>	tmp = string.split(fileBlock.get('blockName'),"/")
119	>	if (len(tmp)==4): del tmp[2]
120	>	blockName=string.join(tmp,"/")
121	>	#print "TMP ",blockName
122	>	# end hack
123	>
124	>	nevtsbyblock[blockName]=nevts
125	>	pass
126	>	except dbsApi.DbsApiException, ex:
127	>	raise DBSError(ex.getClassName(),ex.getErrorMessage())
128	>
129	>	# returning a map of fileblock-nevts  will be enough for now
130	>	# TODO: in future the EvC collections grouped by fileblock should be returned
131	>	return nevtsbyblock
132	>
133	>
134	>	def getDatasetFileBlocks(self, path):
135	>	""" Query DBS to get files/fileblocks """
136	>	try:
137	>	FilesbyBlock={}
138	>	for fileBlock in self.api.getDatasetFileBlocks(path):
139	>	blockname=fileBlock.get('blockName')
140	>	filesinblock=[]
141	>	for files in fileBlock.get('fileList'):
142	>	#print "  block %s has file %s"%(blockname,files.getLogicalFileName())
143	>	filesinblock.append(files.get('logicalFileName'))
144	>	FilesbyBlock[blockname]=filesinblock
145	>	except dbsApi.DbsApiException, ex:
146	>	raise DBSError(ex.getClassName(),ex.getErrorMessage())
147
148	+	return FilesbyBlock

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines