ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/Publisher.py
(Generate patch)

Comparing COMP/CRAB/python/Publisher.py (file contents):
Revision 1.13 by spiga, Thu Jun 19 14:29:18 2008 UTC vs.
Revision 1.21 by fanzago, Tue Jan 20 11:20:46 2009 UTC

# Line 6 | Line 6 | from crab_util import *
6   from crab_logger import Logger
7   from crab_exceptions import *
8   from ProdCommon.FwkJobRep.ReportParser import readJobReport
9 + from ProdCommon.FwkJobRep.ReportState import checkSuccess
10   from ProdCommon.MCPayloads.WorkflowSpec import WorkflowSpec
11   from ProdCommon.DataMgmt.DBS.DBSWriter import DBSWriter
12   from ProdCommon.DataMgmt.DBS.DBSErrors import DBSWriterError, formatEx,DBSReaderError
# Line 24 | Line 25 | class Publisher(Actor):
25          """
26  
27          try:
28 <            self.processedData = cfg_params['USER.publish_data_name']
28 >            self.userprocessedData = cfg_params['USER.publish_data_name']
29 >            self.processedData = None
30          except KeyError:
31              raise CrabException('Cannot publish output data, because you did not specify USER.publish_data_name parameter in the crab.cfg file')
32  
# Line 132 | Line 134 | class Publisher(Actor):
134          datasets=fileinfo.dataset
135          common.logger.debug(6,"FileInfo = " + str(fileinfo))
136          common.logger.debug(6,"DatasetInfo = " + str(datasets))
137 +        if len(datasets)<=0:
138 +           self.exit_status = '1'
139 +           msg = "Error: No info about dataset in the xml file "+file
140 +           common.logger.message(msg)
141 +           return self.exit_status
142          for dataset in datasets:
143              #### for production data
144 +            self.processedData = dataset['ProcessedDataset']
145              if (dataset['PrimaryDataset'] == 'null'):
146 <                dataset['PrimaryDataset'] = dataset['ProcessedDataset']
147 <                
146 >                #dataset['PrimaryDataset'] = dataset['ProcessedDataset']
147 >                dataset['PrimaryDataset'] = self.userprocessedData
148 >            #else: # add parentage from input dataset
149 >            elif self.datasetpath.upper() != 'NONE':
150 >                dataset['ParentDataset']= self.datasetpath
151 >    
152              dataset['PSetContent']=self.content
153              cfgMeta = {'name' : self.pset , 'Type' : 'user' , 'annotation': 'user cfg', 'version' : 'private version'} # add real name of user cfg
154              common.logger.message("PrimaryDataset = %s"%dataset['PrimaryDataset'])
# Line 182 | Line 194 | class Publisher(Actor):
194                  self.noLFN.append(file['PFN'])
195              else:
196                  if int(file['TotalEvents']) != 0 :
197 <                    file.lumisections = {}
197 >                    #file.lumisections = {}
198 >                    # lumi info are now in run hash
199 >                    file.runs = {}
200                      for ds in file.dataset:
187 –                        ds['ProcessedDataset']=procdataset
201                          ### Fede for production
202                          if (ds['PrimaryDataset'] == 'null'):
203 <                            ds['PrimaryDataset']=procdataset
203 >                            #ds['PrimaryDataset']=procdataset
204 >                            ds['PrimaryDataset']=self.userprocessedData
205                      filestopublish.append(file)
206                  else:
207                      self.noEventsFiles.append(file['LFN'])
# Line 202 | Line 216 | class Publisher(Actor):
216          Blocks=None
217          try:
218              Blocks=dbswriter.insertFiles(jobReport)
219 <            common.logger.message("Blocks = %s"%Blocks)
219 >            common.logger.message("Inserting file in blocks = %s"%Blocks)
220          except DBSWriterError, ex:
221 <            common.logger.message("Insert file error: %s"%ex)
221 >            common.logger.error("Insert file error: %s"%ex)
222          return Blocks
223  
224      def run(self):
# Line 213 | Line 227 | class Publisher(Actor):
227          """
228          
229          file_list = glob.glob(self.resDir+"crab_fjr*.xml")
230 +        ## Select only those fjr that are succesfull
231 +        good_list=[]
232 +        for fjr in file_list:
233 +            reports = readJobReport(fjr)
234 +            if len(reports)>0:
235 +               if reports[0].status == "Success":
236 +                  good_list.append(fjr)
237 +        file_list=good_list
238 +        ##
239          common.logger.debug(6, "file_list = "+str(file_list))
240          common.logger.debug(6, "len(file_list) = "+str(len(file_list)))
241              
# Line 227 | Line 250 | class Publisher(Actor):
250  
251              common.logger.message("--->>> Start files publication")
252              for file in file_list:
253 <                common.logger.message("file = "+file)
253 >                common.logger.debug(1, "file = "+file)
254                  Blocks=self.publishAJobReport(file,self.processedData)
255                  if Blocks:
256 <                    [BlocksList.append(x) for x in Blocks]
256 >                    for x in Blocks: # do not allow multiple entries of the same block
257 >                        if x not in BlocksList:
258 >                           BlocksList.append(x)
259                      
260              # close the blocks
261              common.logger.debug(6, "BlocksList = %s"%BlocksList)

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines