| 13 |
|
from WMCore.DataStructs.Workflow import Workflow |
| 14 |
|
from WMCore.JobSplitting.SplitterFactory import SplitterFactory |
| 15 |
|
from WMCore.SiteScreening.BlackWhiteListParser import SEBlackWhiteListParser |
| 16 |
< |
from LumiList import LumiList |
| 16 |
> |
try: # Can remove when CMSSW 3.7 and earlier are dropped |
| 17 |
> |
from FWCore.PythonUtilities.LumiList import LumiList |
| 18 |
> |
except ImportError: |
| 19 |
> |
from LumiList import LumiList |
| 20 |
|
|
| 21 |
|
class JobSplitter: |
| 22 |
|
def __init__( self, cfg_params, args ): |
| 664 |
|
so the job will have AT LEAST as many lumis as requested, perhaps |
| 665 |
|
more |
| 666 |
|
""" |
| 667 |
< |
|
| 667 |
> |
self.useParent = int(self.cfg_params.get('CMSSW.use_parent',0)) |
| 668 |
|
common.logger.debug('Splitting by Lumi') |
| 669 |
|
self.checkLumiSettings() |
| 670 |
|
|
| 672 |
|
pubdata = self.args['pubdata'] |
| 673 |
|
|
| 674 |
|
lumisPerFile = pubdata.getLumis() |
| 675 |
< |
|
| 675 |
> |
self.parentFiles=pubdata.getParent() |
| 676 |
|
# Make the list of WMBS files for job splitter |
| 677 |
|
fileList = pubdata.getListFiles() |
| 678 |
< |
thefiles = Fileset(name='FilesToSplit') |
| 678 |
> |
wmFileList = [] |
| 679 |
|
for jobFile in fileList: |
| 680 |
|
block = jobFile['Block']['Name'] |
| 681 |
|
try: |
| 689 |
|
wmbsFile['block'] = block |
| 690 |
|
for lumi in lumisPerFile[jobFile['LogicalFileName']]: |
| 691 |
|
wmbsFile.addRun(Run(lumi[0], lumi[1])) |
| 692 |
< |
thefiles.addFile(wmbsFile) |
| 692 |
> |
wmFileList.append(wmbsFile) |
| 693 |
> |
|
| 694 |
> |
fileSet = set(wmFileList) |
| 695 |
> |
thefiles = Fileset(name='FilesToSplit', files = fileSet) |
| 696 |
|
|
| 697 |
|
# Create the factory and workflow |
| 698 |
|
work = Workflow() |
| 714 |
|
for jobGroup in jobFactory(lumis_per_job = self.lumisPerJob): |
| 715 |
|
for job in jobGroup.jobs: |
| 716 |
|
if (self.limitNJobs and jobCount >= self.theNumberOfJobs): |
| 717 |
< |
common.logger.info('Limit on number of jobs reached.') |
| 717 |
> |
common.logger.info('Requested number of jobs reached.') |
| 718 |
|
break |
| 719 |
|
if (self.limitTotalLumis and lumisCreated >= self.totalNLumis): |
| 720 |
< |
common.logger.info('Limit on number of lumis reached.') |
| 720 |
> |
common.logger.info('Requested number of lumis reached.') |
| 721 |
|
break |
| 722 |
|
lumis = [] |
| 723 |
|
lfns = [] |
| 724 |
+ |
if self.useParent==1: |
| 725 |
+ |
parentlfns = [] |
| 726 |
+ |
pString ="" |
| 727 |
+ |
|
| 728 |
|
locations = [] |
| 729 |
|
blocks = [] |
| 730 |
|
firstFile = True |
| 747 |
|
lumis.append( (theRun, theLumi) ) |
| 748 |
|
if doFile: |
| 749 |
|
lfns.append(jobFile['lfn']) |
| 750 |
+ |
if self.useParent==1: |
| 751 |
+ |
parent = self.parentFiles[jobFile['lfn']] |
| 752 |
+ |
for p in parent : |
| 753 |
+ |
pString += p + ',' |
| 754 |
|
fileString = ','.join(lfns) |
| 755 |
|
lumiLister = LumiList(lumis = lumis) |
| 756 |
|
lumiString = lumiLister.getCMSSWString() |
| 757 |
< |
list_of_lists.append([fileString, str(-1), str(0), lumiString]) |
| 757 |
> |
if self.useParent==1: |
| 758 |
> |
common.logger.debug("Files: "+fileString+" with the following parents: "+pString[:-1]) |
| 759 |
> |
pfileString = pString[:-1] |
| 760 |
> |
list_of_lists.append([fileString, pfileString, str(-1), str(0), lumiString]) |
| 761 |
> |
else: |
| 762 |
> |
list_of_lists.append([fileString, str(-1), str(0), lumiString]) |
| 763 |
|
list_of_blocks.append(blocks) |
| 764 |
|
jobDestination.append(locations) |
| 765 |
|
jobCount += 1 |
| 772 |
|
# Prepare dict output matching back to non-WMBS job creation |
| 773 |
|
dictOut = {} |
| 774 |
|
dictOut['params'] = ['InputFiles', 'MaxEvents', 'SkipEvents', 'Lumis'] |
| 775 |
+ |
if self.useParent==1: |
| 776 |
+ |
dictOut['params']= ['InputFiles','ParentFiles','MaxEvents','SkipEvents','Lumis'] |
| 777 |
|
dictOut['args'] = list_of_lists |
| 778 |
|
dictOut['jobDestination'] = jobDestination |
| 779 |
|
dictOut['njobs'] = jobCount |
