ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/DBSInfo.py
Revision: 1.14
Committed: Wed Feb 7 00:15:38 2007 UTC (18 years, 2 months ago) by gutsche
Content type: text/x-python
Branch: MAIN
CVS Tags: CRAB_2_4_0_test, CRAB_2_3_1, CRAB_2_3_1_pre6, CRAB_2_3_1_pre5, CRAB_2_3_1_pre4, CRAB_2_3_1_pre3, CRAB_2_3_1_pre2, CRAB_2_3_1_pre1, CRAB_2_3_0, CRAB_2_3_0_pre6, CRAB_2_3_0_pre1, CRAB_2_2_2_pre5, CRAB_2_2_2_pre4, CRAB_2_2_2_pre3, CRAB_2_2_2_pre2, CRAB_2_2_2_pre1, CRAB_2_2_1, CRAB_2_2_1_pre6, CRAB_2_2_1_pre5, CRAB_2_2_1_pre4, PRODCOMMON_0_10_7_testCS2, CRAB_2_2_1_pre3, CRAB_2_2_1_pre2, CRAB_2_2_1_pre1, CRAB_2_2_0, CRAB_2_2_0_pre21, CRAB_2_2_0_pre19, CRAB_2_2_0_pre18, CRAB_2_2_0_pre17, CRAB_2_2_0_pre16, CRAB_2_2_0_pre15, CRAB_2_2_0_pre13, CRAB_2_2_0_pre12, CRAB_2_2_0_pre11, CRAB_2_2_0_pre10, bp_osg_bdii, CRAB_2_2_0_pre9, CRAB_2_2_0_pre8, CRAB_2_2_0_pre7, CRAB_2_1_2, CRAB_2_2_0_pre5, CRAB_2_1_2_pre2, CRAB_2_1_2_pre1, CRAB_2_2_0_pre4, CRAB_2_2_0_pre2, CRAB_2_1_1, CRAB_2_1_1_pre3, CRAB_2_2_0_pre1, CRAB_2_1_1_pre1, CRAB_2_1_0, CRAB_2_1_0_pre6, CRAB_2_1_0_pre5, CRAB_2_1_0_pre4, CRAB_2_1_0_pre3, CRAB_2_1_0_pre2, CRAB_2_1_0_pre1, CRAB_2_0_4, CRAB_2_0_4_pre2, CRAB_2_0_4_pre1, CRAB_2_0_3, CRAB_2_0_3_pre1, CRAB_2_0_2, CRAB_2_0_2_pre6, CRAB_2_0_2_pre5, CRAB_2_0_2_pre4, CRAB_2_0_2_pre3, CRAB_1_5_4_SLC3, CRAB_1_5_4_SLC3_pre4, CRAB_2_0_2_pre2, CRAB_2_0_2_pre1, CRAB_1_5_4_SLC3_pre3, CRAB_2_0_1, CRAB_1_5_4_SLC3_pre2, CRAB_2_0_1_pre1, CRAB_1_5_4_SLC3_pre1, CRAB_2_0_0, CRAB_2_0_0_pre10, CRAB_2_0_0_pre9, CRAB_1_5_4, CRAB_1_5_4_pre2, CRAB_1_5_4_pre1, CRAB_2_0_0_pre7, CRAB_2_0_0_pre6, CRAB_1_5_3, CRAB_1_5_3_pre5, CRAB_1_5_3_pre4, CRAB_2_0_0_pre5, CRAB_1_5_3_pre3, configure, CRAB_2_0_0_pre4, CRAB_1_5_3_pre2, CRAB_1_5_3_pre1, CRAB_2_0_0_pre3, CRAB_1_5_2, CRAB_2_0_0_pre2, CRAB_2_0_0_pre1, CRAB_1_5_1, CRAB_1_5_1_pre4, CRAB_1_5_1_pre3, CRAB_1_5_1_pre2, CRAB_1_5_1_pre1, CRAB_1_5_0, CRAB_1_5_0_pre9, CRAB_1_5_0_pre8, CRAB_1_5_0_pre7
Branch point for: CRAB_2_3_0_br, osg_bdii, CRAB_2_1_2_br, CRAB_2_1_1_pre2, CRAB_1_5_4_SLC3_pre4_br, CRAB_1_5_4_SLC3_start
Changes since 1.13: +16 -16 lines
Log Message: 
transition to new DBS api which follows python namespaces

File Contents

# Content
1 #!/usr/bin/env python
2 import sys, os, string, commands
3 import exceptions
4 import common
5 from crab_exceptions import *
6 import DBSAPIOLD.dbsCgiApi
7 import DBSAPIOLD.dbsApi
8
9 ## for python 2.2 add the pyexpat.so to PYTHONPATH
10 pythonV=sys.version.split(' ')[0]
11 if pythonV.find('2.2') >= 0 :
12 Crabpydir=commands.getoutput('which crab')
13 Topdir=string.replace(Crabpydir,'/python/crab','')
14 extradir=Topdir+'/DLSAPI/extra'
15 if sys.path.count(extradir) <= 0:
16 if os.path.exists(extradir):
17 sys.path.insert(0, extradir)
18
19 # #######################################
20 class DBSError(exceptions.Exception):
21 def __init__(self, errorName, errorMessage):
22 self.args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
23 exceptions.Exception.__init__(self, self.args)
24 pass
25
26 def getErrorMessage(self):
27 """ Return error message """
28 return "%s" % (self.args)
29
30 # #######################################
31 class DBSInvalidDataTierError(exceptions.Exception):
32 def __init__(self, errorName, errorMessage):
33 self.args='\nERROR DBS %s : %s \n'%(errorName,errorMessage)
34 exceptions.Exception.__init__(self, self.args)
35 pass
36
37 def getErrorMessage(self):
38 """ Return error message """
39 return "%s" % (self.args)
40
41 # #######################################
42 class DBSInfoError:
43 def __init__(self, url):
44 print '\nERROR accessing DBS url : '+url+'\n'
45 pass
46
47 ##################################################################################
48 # Class to extract info from DBS
49 ###############################################################################
50
51 class DBSInfo:
52 def __init__(self, dbs_url, dbs_instance):
53 """
54 Construct api object.
55 """
56 ## cgi service API
57 args = {}
58 args['instance']=dbs_instance
59
60 common.logger.debug(3,"Accessing DBS at: "+dbs_url+" "+dbs_instance)
61
62 self.api = DBSAPIOLD.dbsCgiApi.DbsCgiApi(dbs_url, args)
63 ## set log level
64 # self.api.setLogLevel(DBSAPIOLD.dbsApi.DBS_LOG_LEVEL_INFO_)
65 #self.api.setLogLevel(DBSAPIOLD.dbsApi.DBS_LOG_LEVEL_QUIET_)
66
67 def getMatchingDatasets (self, datasetPath):
68 """ Query DBS to get provenance """
69 try:
70 result = self.api.listProcessedDatasets("%s" %datasetPath)
71 except DBSAPIOLD.dbsApi.InvalidDataTier, ex:
72 raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
73 except DBSAPIOLD.dbsApi.DbsApiException, ex:
74 raise DBSError(ex.getClassName(),ex.getErrorMessage())
75 except DBSAPIOLD.dbsCgiApi.DbsCgiToolError , ex:
76 raise DBSError(ex.getClassName(),ex.getErrorMessage())
77 except DBSAPIOLD.dbsCgiApi.DbsCgiBadResponse , ex:
78 raise DBSError(ex.getClassName(),ex.getErrorMessage())
79
80 return result
81
82
83 def getDatasetProvenance(self, path, dataTiers):
84 """ Query DBS to get provenance """
85 try:
86 datasetParentList = self.api.getDatasetProvenance(path,dataTiers)
87 except DBSAPIOLD.dbsApi.InvalidDataTier, ex:
88 raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage())
89 except DBSAPIOLD.dbsApi.DbsApiException, ex:
90 raise DBSError(ex.getClassName(),ex.getErrorMessage())
91 return datasetParentList
92
93 def getEventsPerBlock(self, path):
94 """ Query DBS to get event collections """
95 # count events per block
96 nevtsbyblock = {}
97 try:
98 contents = self.api.getDatasetContents(path)
99 except DBSAPIOLD.dbsApi.DbsApiException, ex:
100 raise DBSError(ex.getClassName(),ex.getErrorMessage())
101 except DBSAPIOLD.dbsCgiApi.DbsCgiBadResponse, ex:
102 raise DBSError(ex.getClassName(),ex.getErrorMessage())
103 for fileBlock in contents:
104 ## get the event collections for each block
105 nevts = 0
106 eventCollectionList = fileBlock.get('eventCollectionList')
107 for evc in eventCollectionList:
108 nevts = nevts + evc.get('numberOfEvents')
109
110 common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(nevts,fileBlock.get('blockName')))
111 nevtsbyblock[fileBlock.get('blockName')]=nevts
112
113 # returning a map of fileblock-nevts will be enough for now
114 # TODO: in future the EvC collections grouped by fileblock should be returned
115 return nevtsbyblock
116
117 def getEventsPerFile(self, path):
118 """ Query DBS to get a dictionary of files:(events/file) """
119 numEventsByFile = {}
120 try:
121 contents = self.api.getDatasetContents(path)
122 except DBSAPIOLD.dbsApi.DbsApiException, ex:
123 raise DBSError(ex.getClassName(),ex.getErrorMessage())
124 for fileBlock in contents:
125 numEvents = 0
126 eventCollectionList = fileBlock.get('eventCollectionList')
127 for evc in eventCollectionList:
128 numEvents = evc.get('numberOfEvents')
129 fileList = evc.get('fileList')
130 # As of 2006-08-10, event collections contain only one file
131 # => fileList contains only one dictionary
132 if len(fileList)>1:
133 msg = "Event collection contains more than one file! Exiting.\n"
134 msg = msg + "CRAB and DBS must be upgraded to handle event collections with multiple files.\n"
135 raise CrabException(msg)
136 fileDict = fileList[0]
137 fileName = fileDict.get('logicalFileName')
138 numEventsByFile[fileName] = numEvents
139 return numEventsByFile
140
141 def getDatasetFileBlocks(self, path):
142 """ Query DBS to get files/fileblocks """
143 try:
144 FilesbyBlock={}
145 try:
146 allBlocks = self.api.getDatasetFileBlocks(path)
147 except DBSAPIOLD.dbsCgiApi.DbsCgiBadResponse, ex:
148 raise DBSError(ex.getClassName(), ex.getErrorMessage())
149 for fileBlock in allBlocks:
150 blockname=fileBlock.get('blockName')
151 filesinblock=[]
152 for files in fileBlock.get('fileList'):
153 #print " block %s has file %s"%(blockname,files.getLogicalFileName())
154 filesinblock.append(files.get('logicalFileName'))
155 FilesbyBlock[blockname]=filesinblock
156 except DBSAPIOLD.dbsApi.DbsApiException, ex:
157 raise DBSError(ex.getClassName(),ex.getErrorMessage())
158
159 return FilesbyBlock