ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/Splitter.py
(Generate patch)

Comparing COMP/CRAB/python/Splitter.py (file contents):
Revision 1.46 by spiga, Mon Sep 27 08:56:08 2010 UTC vs.
Revision 1.51 by spiga, Mon May 9 10:42:08 2011 UTC

# Line 279 | Line 279 | class JobSplitter:
279                                  fullString = parString[:-1]
280                                  if self.useParent==1:
281                                      fullParentString = pString[:-1]
282 <                                    list_of_lists.append([fullString,fullParentString,str(-1),str(jobSkipEventCount)])
282 >                                    list_of_lists.append([fullString,fullParentString,str(-1),str(jobSkipEventCount),block])
283                                  else:
284 <                                    list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
284 >                                    list_of_lists.append([fullString,str(-1),str(jobSkipEventCount),block])
285                                  msg += "Job %s can run over %s  events (last file in block).\n"%(str(jobCount+1), str(filesEventCount - jobSkipEventCount))
286                                  jobDestination.append(blockSites[block])
287                                  msg += "Job %s Destination: %s\n"%(str(jobCount+1),str(SE2CMS(jobDestination[jobCount])))
# Line 308 | Line 308 | class JobSplitter:
308                          fullString = parString[:-1]
309                          if self.useParent==1:
310                              fullParentString = pString[:-1]
311 <                            list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount)])
311 >                            list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount),block])
312                          else:
313 <                            list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
313 >                            list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount),block])
314                          msg += "Job %s can run over %s events.\n"%(str(jobCount+1),str(eventsPerJobRequested))
315                          jobDestination.append(blockSites[block])
316                          msg+= "Job %s Destination: %s\n"%(str(jobCount+1),str(SE2CMS(jobDestination[jobCount])))
# Line 333 | Line 333 | class JobSplitter:
333                          fullString = parString[:-1]
334                          if self.useParent==1:
335                              fullParentString = pString[:-1]
336 <                            list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount)])
336 >                            list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount),block])
337                          else:
338 <                            list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
338 >                            list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount),block])
339                          msg += "Job %s can run over %s events.\n"%(str(jobCount+1),str(eventsPerJobRequested))
340                          jobDestination.append(blockSites[block])
341                          msg+= "Job %s Destination: %s\n"%(str(jobCount+1),str(SE2CMS(jobDestination[jobCount])))
# Line 368 | Line 368 | class JobSplitter:
368  
369         # prepare dict output
370          dictOut = {}
371 <        dictOut['params']= ['InputFiles','MaxEvents','SkipEvents']
372 <        if self.useParent: dictOut['params']= ['InputFiles','ParentFiles','MaxEvents','SkipEvents']
371 >        dictOut['params']= ['InputFiles','MaxEvents','SkipEvents','InputBlocks']
372 >        if self.useParent: dictOut['params']= ['InputFiles','ParentFiles','MaxEvents','SkipEvents','InputBlocks']
373          dictOut['args'] = list_of_lists
374          dictOut['jobDestination'] = jobDestination
375          dictOut['njobs']=self.total_number_of_jobs
# Line 494 | Line 494 | class JobSplitter:
494                  parString = ''
495                  for file in res['lfns']:
496                      parString += file + ','
497 +                list_of_blocks.append(res['block'])
498                  fullString = parString[:-1]
499 <                list_of_lists.append([fullString,str(-1),str(0)])
499 >                blockString=','.join(list_of_blocks)
500 >                list_of_lists.append([fullString,str(-1),str(0),blockStringa])
501                  #need to check single file location
502                  jobDestination.append(res['locations'])
501 –                list_of_blocks.append(res['block'])
503                  count +=1
504          # prepare dict output
505          dictOut = {}
506 <        dictOut['params']= ['InputFiles','MaxEvents','SkipEvents']
506 >        dictOut['params']= ['InputFiles','MaxEvents','SkipEvents','InputBlocks']
507          dictOut['args'] = list_of_lists
508          dictOut['jobDestination'] = jobDestination
509          dictOut['njobs']=count
509 –
510          self.cacheBlocks(list_of_blocks,jobDestination)
511  
512          return dictOut
# Line 707 | Line 707 | class JobSplitter:
707          lumisCreated = 0
708          list_of_blocks = []
709          if not self.limitJobLumis:
710 <            self.lumisPerJob = pubdata.getMaxLumis() // self.theNumberOfJobs + 1
710 >            if self.totalNLumis > 0:
711 >                self.lumisPerJob = max(self.totalNLumis // self.theNumberOfJobs,1)
712 >            else:
713 >                self.lumisPerJob = pubdata.getMaxLumis() // self.theNumberOfJobs + 1
714              common.logger.info('Each job will process about %s lumis.' %
715                                  self.lumisPerJob)
716  
# Line 722 | Line 725 | class JobSplitter:
725                  lumis = []
726                  lfns  = []
727                  if self.useParent==1:
728 <                 parentlfns  = []  
728 >                 parentlfns  = []
729                   pString =""
730  
731                  locations = []
# Line 754 | Line 757 | class JobSplitter:
757                  fileString = ','.join(lfns)
758                  lumiLister = LumiList(lumis = lumis)
759                  lumiString = lumiLister.getCMSSWString()
760 +                blockString=','.join(blocks)
761                  if self.useParent==1:
762                    common.logger.debug("Files: "+fileString+" with the following parents: "+pString[:-1])
763                    pfileString = pString[:-1]
764 <                  list_of_lists.append([fileString, pfileString, str(-1), str(0), lumiString])
765 <                else:
766 <                 list_of_lists.append([fileString, str(-1), str(0), lumiString])
764 >                  list_of_lists.append([fileString, pfileString, str(-1), str(0), lumiString,blockString])
765 >                else:
766 >                 list_of_lists.append([fileString, str(-1), str(0), lumiString, blockString])
767                  list_of_blocks.append(blocks)
768                  jobDestination.append(locations)
769                  jobCount += 1
# Line 771 | Line 775 | class JobSplitter:
775  
776          # Prepare dict output matching back to non-WMBS job creation
777          dictOut = {}
778 <        dictOut['params'] = ['InputFiles', 'MaxEvents', 'SkipEvents', 'Lumis']
779 <        if self.useParent==1:  
780 <         dictOut['params']= ['InputFiles','ParentFiles','MaxEvents','SkipEvents','Lumis']
778 >        dictOut['params'] = ['InputFiles', 'MaxEvents', 'SkipEvents', 'Lumis','InputBlocks']
779 >        if self.useParent==1:
780 >         dictOut['params']= ['InputFiles','ParentFiles','MaxEvents','SkipEvents','Lumis','InputBlocks']
781          dictOut['args'] = list_of_lists
782          dictOut['jobDestination'] = jobDestination
783          dictOut['njobs'] = jobCount
780 –
784          self.cacheBlocks(list_of_blocks,jobDestination)
785  
786          return dictOut

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines