ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/DataDiscovery.py
(Generate patch)

Comparing COMP/CRAB/python/DataDiscovery.py (file contents):
Revision 1.21 by spiga, Thu Jun 5 07:13:31 2008 UTC vs.
Revision 1.28 by spiga, Fri Feb 6 18:14:58 2009 UTC

# Line 4 | Line 4 | import DBSAPI.dbsApi
4   from DBSAPI.dbsApiException import *
5   import common
6   from crab_util import *
7 + import os
8  
9  
10   # #######################################
# Line 70 | Line 71 | class NoDataTierinProvenanceError(except
71   # ####################################
72   # class to find and extact info from published data
73   class DataDiscovery:
74 <    def __init__(self, datasetPath, cfg_params):
74 >    def __init__(self, datasetPath, cfg_params, skipAnBlocks):
75  
76          #       Attributes
77          self.datasetPath = datasetPath
78          self.cfg_params = cfg_params
79 +        self.skipBlocks = skipAnBlocks
80  
81          self.eventsPerBlock = {}  # DBS output: map fileblocks-events for collection
82          self.eventsPerFile = {}   # DBS output: map files-events
# Line 87 | Line 89 | class DataDiscovery:
89          """
90          Contact DBS
91          """
90 –
92          ## get DBS URL
93 <        dbs_url="http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
94 <        if (self.cfg_params.has_key('CMSSW.dbs_url')):
95 <            dbs_url=self.cfg_params['CMSSW.dbs_url']
93 >        global_url="http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
94 >        caf_url = "http://cmsdbsprod.cern.ch/cms_dbs_caf_analysis_01/servlet/DBSServlet"
95 >        dbs_url_map  =   {'glite':    global_url,
96 >                          'glitecoll':global_url,\
97 >                          'condor':   global_url,\
98 >                          'condor_g': global_url,\
99 >                          'glidein':  global_url,\
100 >                          'lsf':      global_url,\
101 >                          'caf':      caf_url,\
102 >                          'sge':      global_url
103 >                          }
104  
105 +        dbs_url_default = dbs_url_map[(common.scheduler.name()).lower()]
106 +        dbs_url=  self.cfg_params.get('CMSSW.dbs_url', dbs_url_default)
107          common.logger.debug(3,"Accessing DBS at: "+dbs_url)
108  
109          ## check if runs are selected
# Line 100 | Line 111 | class DataDiscovery:
111          if (self.cfg_params.has_key('CMSSW.runselection')):
112              runselection = parseRange2(self.cfg_params['CMSSW.runselection'])
113  
114 +
115 +        self.splitByRun = int(self.cfg_params.get('CMSSW.split_by_run', 0))
116 +          
117 +        self.ads = int(self.cfg_params.get('CMSSW.ads', 0))
118 +
119          common.logger.debug(6,"runselection is: %s"%runselection)
120          ## service API
121          args = {}
# Line 107 | Line 123 | class DataDiscovery:
123          args['level']   = 'CRITICAL'
124  
125          ## check if has been requested to use the parent info
126 <        if (self.cfg_params.has_key('CMSSW.runselection')):
111 <            runselection = parseRange2(self.cfg_params['CMSSW.runselection'])
126 >        useparent = int(self.cfg_params.get('CMSSW.use_parent',0))
127  
128 <        useParent = self.cfg_params.get('CMSSW.use_parent',False)
129 <    
130 <        allowedRetriveValue = [
131 <                        'retrive_child',
117 <                        'retrive_block',
118 <                        'retrive_lumi',
119 <                        'retrive_run'
120 <                        ]
121 <        if useParent:  allowedRetriveValue.append('retrive_parent')
122 <        common.logger.debug(5,"Set of input parameters used for DBS query : \n"+str(allowedRetriveValue))
123 <        common.logger.write("Set of input parameters used for DBS query : \n"+str(allowedRetriveValue))
128 >        ## check if has been asked for a non default file to store/read analyzed fileBlocks  
129 >        defaultName = common.work_space.shareDir()+'AnalyzedBlocks.txt'  
130 >        fileBlocks_FileName = os.path.abspath(self.cfg_params.get('CMSSW.fileblocks_file',defaultName))
131 >
132          api = DBSAPI.dbsApi.DbsApi(args)
133 +
134 +        self.files = self.queryDbs(api,path=self.datasetPath,runselection=runselection,useParent=useparent)
135 +
136 +        anFileBlocks = []
137 +        if self.skipBlocks: anFileBlocks = readTXTfile(self, fileBlocks_FileName)
138 +
139 +        # parse files and fill arrays
140 +        for file in self.files :
141 +            parList = []
142 +            # skip already analyzed blocks
143 +            fileblock = file['Block']['Name']
144 +            if fileblock not in anFileBlocks :
145 +                filename = file['LogicalFileName']
146 +                # asked retry the list of parent for the given child
147 +                if useparent==1: parList = [x['LogicalFileName'] for x in file['ParentList']]
148 +                self.parent[filename] = parList
149 +                if filename.find('.dat') < 0 :
150 +                    events    = file['NumberOfEvents']
151 +                    # number of events per block
152 +                    if fileblock in self.eventsPerBlock.keys() :
153 +                        self.eventsPerBlock[fileblock] += events
154 +                    else :
155 +                        self.eventsPerBlock[fileblock] = events
156 +                    # number of events per file
157 +                    self.eventsPerFile[filename] = events
158 +            
159 +                    # number of events per block
160 +                    if fileblock in self.blocksinfo.keys() :
161 +                        self.blocksinfo[fileblock].append(filename)
162 +                    else :
163 +                        self.blocksinfo[fileblock] = [filename]
164 +            
165 +                    # total number of events
166 +                    self.maxEvents += events
167 +        if  self.skipBlocks and len(self.eventsPerBlock.keys()) == 0:
168 +            msg = "No new fileblocks available for dataset: "+str(self.datasetPath)
169 +            raise  CrabException(msg)    
170 +
171 +        saveFblocks=''
172 +        for block in self.eventsPerBlock.keys() :
173 +            saveFblocks += str(block)+'\n'
174 +            common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(self.eventsPerBlock[block],block))
175 +        writeTXTfile(self, fileBlocks_FileName , saveFblocks)
176 +                      
177 +        if len(self.eventsPerBlock) <= 0:
178 +            raise NotExistingDatasetError(("\nNo data for %s in DBS\nPlease check"
179 +                                            + " dataset path variables in crab.cfg")
180 +                                            % self.datasetPath)
181 +
182 +
183 + ###########################
184 +
185 +    def queryDbs(self,api,path=None,runselection=None,useParent=None):
186 +
187 +        allowedRetriveValue = [#'retrive_parent',
188 +                               'retrive_block',
189 +                               #'retrive_lumi',
190 +                               'retrive_run'
191 +                               ]
192          try:
193 <            if len(runselection) <= 0 :
194 <                files = api.listFiles(path=self.datasetPath,retriveList=allowedRetriveValue)
193 >            if len(runselection) <=0 :
194 >                if useParent==1 or self.splitByRun==1 :
195 >                    if self.ads==1 :          
196 >                        files = api.listFiles(analysisDataset=path, retriveList=allowedRetriveValue)
197 >                    else :
198 >                        files = api.listFiles(path=path, retriveList=allowedRetriveValue)
199 >                    common.logger.debug(5,"Set of input parameters used for DBS query : \n"+str(allowedRetriveValue))
200 >                    common.logger.write("Set of input parameters used for DBS query : \n"+str(allowedRetriveValue))
201 >                else:
202 >                    files = api.listDatasetFiles(self.datasetPath)
203              else :
204                  files=[]
205                  for arun in runselection:
206                      try:
207 <                        filesinrun = api.listFiles(path=self.datasetPath,retriveList=allowedRetriveValue,runNumber=arun)
207 >                        if self.ads==1 : filesinrun = api.listFiles(analysisDataset=path,retriveList=allowedRetriveValue,runNumber=arun)
208 >                        else: filesinrun = api.listFiles(path=path,retriveList=allowedRetriveValue,runNumber=arun)
209                          files.extend(filesinrun)
210                      except:
211                          msg="WARNING: problem extracting info from DBS for run %s "%arun
# Line 141 | Line 217 | class DataDiscovery:
217          except DBSError, msg:
218              raise DataDiscoveryError(msg)
219  
220 <        # parse files and fill arrays
145 <        for file in files :
146 <            parList = []
147 <            filename = file['LogicalFileName']
148 <            # asked retry the list of parent for the given child
149 <            if useParent: parList = [x['LogicalFileName'] for x in file['ParentList']]
150 <            self.parent[filename] = parList
151 <            if filename.find('.dat') < 0 :
152 <                fileblock = file['Block']['Name']
153 <                events    = file['NumberOfEvents']
154 <                # number of events per block
155 <                if fileblock in self.eventsPerBlock.keys() :
156 <                    self.eventsPerBlock[fileblock] += events
157 <                else :
158 <                    self.eventsPerBlock[fileblock] = events
159 <                # number of events per file
160 <                self.eventsPerFile[filename] = events
161 <
162 <                # number of events per block
163 <                if fileblock in self.blocksinfo.keys() :
164 <                    self.blocksinfo[fileblock].append(filename)
165 <                else :
166 <                    self.blocksinfo[fileblock] = [filename]
167 <
168 <                # total number of events
169 <                self.maxEvents += events
170 <
171 <        for block in self.eventsPerBlock.keys() :
172 <            common.logger.debug(6,"DBSInfo: total nevts %i in block %s "%(self.eventsPerBlock[block],block))
173 <
174 <        if len(self.eventsPerBlock) <= 0:
175 <            raise NotExistingDatasetError(("\nNo data for %s in DBS\nPlease check"
176 <                                            + " dataset path variables in crab.cfg")
177 <                                            % self.datasetPath)
178 <
220 >        return files
221  
222   # #################################################
223      def getMaxEvents(self):
# Line 212 | Line 254 | class DataDiscovery:
254          """
255          return self.parent        
256  
257 + # #################################################
258 +    def getListFiles(self):
259 +        """
260 +        return parent grouped by file
261 +        """
262 +        return self.files        
263 +
264   ########################################################################

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines