ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/Splitter.py
(Generate patch)

Comparing COMP/CRAB/python/Splitter.py (file contents):
Revision 1.46 by spiga, Mon Sep 27 08:56:08 2010 UTC vs.
Revision 1.54 by spiga, Tue Mar 6 19:22:59 2012 UTC

# Line 279 | Line 279 | class JobSplitter:
279                                  fullString = parString[:-1]
280                                  if self.useParent==1:
281                                      fullParentString = pString[:-1]
282 <                                    list_of_lists.append([fullString,fullParentString,str(-1),str(jobSkipEventCount)])
282 >                                    list_of_lists.append([fullString,fullParentString,str(-1),str(jobSkipEventCount),block])
283                                  else:
284 <                                    list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
284 >                                    list_of_lists.append([fullString,str(-1),str(jobSkipEventCount),block])
285                                  msg += "Job %s can run over %s  events (last file in block).\n"%(str(jobCount+1), str(filesEventCount - jobSkipEventCount))
286                                  jobDestination.append(blockSites[block])
287                                  msg += "Job %s Destination: %s\n"%(str(jobCount+1),str(SE2CMS(jobDestination[jobCount])))
# Line 308 | Line 308 | class JobSplitter:
308                          fullString = parString[:-1]
309                          if self.useParent==1:
310                              fullParentString = pString[:-1]
311 <                            list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount)])
311 >                            list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount),block])
312                          else:
313 <                            list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
313 >                            list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount),block])
314                          msg += "Job %s can run over %s events.\n"%(str(jobCount+1),str(eventsPerJobRequested))
315                          jobDestination.append(blockSites[block])
316                          msg+= "Job %s Destination: %s\n"%(str(jobCount+1),str(SE2CMS(jobDestination[jobCount])))
# Line 333 | Line 333 | class JobSplitter:
333                          fullString = parString[:-1]
334                          if self.useParent==1:
335                              fullParentString = pString[:-1]
336 <                            list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount)])
336 >                            list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount),block])
337                          else:
338 <                            list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
338 >                            list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount),block])
339                          msg += "Job %s can run over %s events.\n"%(str(jobCount+1),str(eventsPerJobRequested))
340                          jobDestination.append(blockSites[block])
341                          msg+= "Job %s Destination: %s\n"%(str(jobCount+1),str(SE2CMS(jobDestination[jobCount])))
# Line 368 | Line 368 | class JobSplitter:
368  
369         # prepare dict output
370          dictOut = {}
371 <        dictOut['params']= ['InputFiles','MaxEvents','SkipEvents']
372 <        if self.useParent: dictOut['params']= ['InputFiles','ParentFiles','MaxEvents','SkipEvents']
371 >        dictOut['params']= ['InputFiles','MaxEvents','SkipEvents','InputBlocks']
372 >        if self.useParent: dictOut['params']= ['InputFiles','ParentFiles','MaxEvents','SkipEvents','InputBlocks']
373          dictOut['args'] = list_of_lists
374          dictOut['jobDestination'] = jobDestination
375          dictOut['njobs']=self.total_number_of_jobs
# Line 428 | Line 428 | class JobSplitter:
428              common.logger.info(msg)
429  
430          if bloskNoSite == allBlock:
431 <            msg += 'Requested jobs cannot be Created! \n'
431 >            msg = 'Requested jobs cannot be Created! \n'
432              if self.cfg_params.has_key('GRID.se_white_list'):
433                  msg += '\tWARNING: SE White List: '+self.cfg_params['GRID.se_white_list']+'\n'
434                  msg += '\t(Hint: By whitelisting you force the job to run at this particular site(s).\n'
# Line 464 | Line 464 | class JobSplitter:
464              except:
465                  continue
466              wmbsFile = File(f['LogicalFileName'])
467 +            if not  blockSites[block]:
468 +                msg = 'WARNING: No sites are hosting any part of data for block: %s\n' %block                
469 +                msg += 'Related jobs will not be submitted and this block of data can not be analyzed'
470 +                common.logger.debug(msg)
471              [ wmbsFile['locations'].add(x) for x in blockSites[block] ]
472              wmbsFile['block'] = block
473              runNum = f['RunsList'][0]['RunNumber']
# Line 494 | Line 498 | class JobSplitter:
498                  parString = ''
499                  for file in res['lfns']:
500                      parString += file + ','
501 +                list_of_blocks.append(res['block'])
502                  fullString = parString[:-1]
503 <                list_of_lists.append([fullString,str(-1),str(0)])
503 >                blockString=','.join(list_of_blocks)
504 >                list_of_lists.append([fullString,str(-1),str(0),blockString])
505                  #need to check single file location
506                  jobDestination.append(res['locations'])
501 –                list_of_blocks.append(res['block'])
507                  count +=1
508          # prepare dict output
509          dictOut = {}
510 <        dictOut['params']= ['InputFiles','MaxEvents','SkipEvents']
510 >        dictOut['params']= ['InputFiles','MaxEvents','SkipEvents','InputBlocks']
511          dictOut['args'] = list_of_lists
512          dictOut['jobDestination'] = jobDestination
513          dictOut['njobs']=count
509 –
514          self.cacheBlocks(list_of_blocks,jobDestination)
515  
516          return dictOut
# Line 684 | Line 688 | class JobSplitter:
688                  continue
689              wmbsFile = File(jobFile['LogicalFileName'])
690              if not  blockSites[block]:
691 <                wmbsFile['locations'].add('Nowhere')
691 >                msg = 'WARNING: No sites are hosting any part of data for block: %s\n' %block
692 >                msg += 'Related jobs will not be submitted and this block of data can not be analyzed'
693 >                common.logger.debug(msg)
694 >               # wmbsFile['locations'].add('Nowhere')
695              [ wmbsFile['locations'].add(x) for x in blockSites[block] ]
696              wmbsFile['block'] = block
697              for lumi in lumisPerFile[jobFile['LogicalFileName']]:
# Line 707 | Line 714 | class JobSplitter:
714          lumisCreated = 0
715          list_of_blocks = []
716          if not self.limitJobLumis:
717 <            self.lumisPerJob = pubdata.getMaxLumis() // self.theNumberOfJobs + 1
717 >            if self.totalNLumis > 0:
718 >                self.lumisPerJob = max(self.totalNLumis // self.theNumberOfJobs,1)
719 >            else:
720 >                self.lumisPerJob = pubdata.getMaxLumis() // self.theNumberOfJobs + 1
721              common.logger.info('Each job will process about %s lumis.' %
722                                  self.lumisPerJob)
723  
# Line 722 | Line 732 | class JobSplitter:
732                  lumis = []
733                  lfns  = []
734                  if self.useParent==1:
735 <                 parentlfns  = []  
735 >                 parentlfns  = []
736                   pString =""
737  
738                  locations = []
# Line 754 | Line 764 | class JobSplitter:
764                  fileString = ','.join(lfns)
765                  lumiLister = LumiList(lumis = lumis)
766                  lumiString = lumiLister.getCMSSWString()
767 +                blockString=','.join(blocks)
768                  if self.useParent==1:
769                    common.logger.debug("Files: "+fileString+" with the following parents: "+pString[:-1])
770                    pfileString = pString[:-1]
771 <                  list_of_lists.append([fileString, pfileString, str(-1), str(0), lumiString])
772 <                else:
773 <                 list_of_lists.append([fileString, str(-1), str(0), lumiString])
771 >                  list_of_lists.append([fileString, pfileString, str(-1), str(0), lumiString,blockString])
772 >                else:
773 >                 list_of_lists.append([fileString, str(-1), str(0), lumiString, blockString])
774                  list_of_blocks.append(blocks)
775                  jobDestination.append(locations)
776                  jobCount += 1
# Line 771 | Line 782 | class JobSplitter:
782  
783          # Prepare dict output matching back to non-WMBS job creation
784          dictOut = {}
785 <        dictOut['params'] = ['InputFiles', 'MaxEvents', 'SkipEvents', 'Lumis']
786 <        if self.useParent==1:  
787 <         dictOut['params']= ['InputFiles','ParentFiles','MaxEvents','SkipEvents','Lumis']
785 >        dictOut['params'] = ['InputFiles', 'MaxEvents', 'SkipEvents', 'Lumis','InputBlocks']
786 >        if self.useParent==1:
787 >         dictOut['params']= ['InputFiles','ParentFiles','MaxEvents','SkipEvents','Lumis','InputBlocks']
788          dictOut['args'] = list_of_lists
789          dictOut['jobDestination'] = jobDestination
790          dictOut['njobs'] = jobCount
780 –
791          self.cacheBlocks(list_of_blocks,jobDestination)
792  
793          return dictOut

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines