| 3 |
|
__version__ = "$Revision$" |
| 4 |
|
|
| 5 |
|
import common |
| 6 |
– |
from sets import Set |
| 6 |
|
from crab_exceptions import * |
| 7 |
|
from crab_util import * |
| 8 |
|
|
| 13 |
|
from WMCore.DataStructs.Workflow import Workflow |
| 14 |
|
from WMCore.JobSplitting.SplitterFactory import SplitterFactory |
| 15 |
|
from WMCore.SiteScreening.BlackWhiteListParser import SEBlackWhiteListParser |
| 16 |
+ |
from LumiList import LumiList |
| 17 |
|
|
| 18 |
|
class JobSplitter: |
| 19 |
|
def __init__( self, cfg_params, args ): |
| 85 |
|
settings += 1 |
| 86 |
|
|
| 87 |
|
if settings != 2: |
| 88 |
< |
msg = 'When running on analysis datasets you must specify two and only two of:\n' |
| 88 |
> |
msg = 'When splitting by lumi section you must specify two and only two of:\n' |
| 89 |
|
msg += ' number_of_jobs, lumis_per_job, total_number_of_lumis' |
| 90 |
|
raise CrabException(msg) |
| 91 |
|
if self.limitNJobs and self.limitJobLumis: |
| 411 |
|
common.logger.info(msg) |
| 412 |
|
|
| 413 |
|
if bloskNoSite == allBlock: |
| 414 |
< |
raise CrabException('No jobs created') |
| 414 |
> |
msg += 'Requested jobs cannot be Created! \n' |
| 415 |
> |
if self.cfg_params.has_key('GRID.se_white_list'): |
| 416 |
> |
msg += '\tWARNING: SE White List: '+self.cfg_params['GRID.se_white_list']+'\n' |
| 417 |
> |
msg += '\t(Hint: By whitelisting you force the job to run at this particular site(s).\n' |
| 418 |
> |
msg += '\tPlease check if the dataset is available at this site!)' |
| 419 |
> |
if self.cfg_params.has_key('GRID.ce_white_list'): |
| 420 |
> |
msg += '\tWARNING: CE White List: '+self.cfg_params['GRID.ce_white_list']+'\n' |
| 421 |
> |
msg += '\t(Hint: By whitelisting you force the job to run at this particular site(s).\n' |
| 422 |
> |
msg += '\tPlease check if the dataset is available at this site!)\n' |
| 423 |
> |
raise CrabException(msg) |
| 424 |
|
|
| 425 |
|
return |
| 426 |
|
|
| 467 |
|
jobfactory = splitter(subs) |
| 468 |
|
|
| 469 |
|
#loop over all runs |
| 461 |
– |
set = Set(runList) |
| 470 |
|
list_of_lists = [] |
| 471 |
|
jobDestination = [] |
| 472 |
|
count = 0 |
| 547 |
|
|
| 548 |
|
managedGenerators =self.args['managedGenerators'] |
| 549 |
|
generator = self.args['generator'] |
| 550 |
< |
firstRun = self.cfg_params.get('CMSSW.first_run', 1) |
| 550 |
> |
firstLumi = self.cfg_params.get('CMSSW.first_lumi', 1) |
| 551 |
|
|
| 552 |
|
self.prepareSplittingNoInput() |
| 553 |
|
|
| 568 |
|
## Since there is no input, any site is good |
| 569 |
|
jobDestination.append([""]) # must be empty to correctly write the XML |
| 570 |
|
args=[] |
| 571 |
< |
if (firstRun): # Pythia first run |
| 572 |
< |
args.append(str(int(firstRun)+i)) |
| 571 |
> |
if (firstLumi): # Pythia first lumi |
| 572 |
> |
args.append(str(int(firstLumi)+i)) |
| 573 |
|
if (generator in managedGenerators): |
| 574 |
|
args.append(generator) |
| 575 |
|
if (generator == 'comphep' and i == 0): |
| 583 |
|
|
| 584 |
|
dictOut = {} |
| 585 |
|
dictOut['params'] = ['MaxEvents'] |
| 586 |
< |
if (firstRun): |
| 587 |
< |
dictOut['params'] = ['FirstRun','MaxEvents'] |
| 588 |
< |
if ( generator in managedGenerators ) : |
| 589 |
< |
dictOut['params'] = ['FirstRun', 'Generator', 'FirstEvent', 'MaxEvents'] |
| 586 |
> |
if (firstLumi): |
| 587 |
> |
dictOut['params'] = ['FirstLumi','MaxEvents'] |
| 588 |
> |
if (generator in managedGenerators): |
| 589 |
> |
dictOut['params'] = ['FirstLumi', 'Generator', 'FirstEvent', 'MaxEvents'] |
| 590 |
|
else: |
| 591 |
|
if (generator in managedGenerators) : |
| 592 |
|
dictOut['params'] = ['Generator', 'FirstEvent', 'MaxEvents'] |
| 699 |
|
firstFile = True |
| 700 |
|
# Collect information from all the files |
| 701 |
|
for jobFile in job.getFiles(): |
| 702 |
+ |
doFile = False |
| 703 |
|
if firstFile: # Get locations from first file in the job |
| 704 |
|
for loc in jobFile['locations']: |
| 705 |
|
locations.append(loc) |
| 708 |
|
for lumiList in jobFile['runs']: |
| 709 |
|
theRun = lumiList.run |
| 710 |
|
for theLumi in list(lumiList): |
| 711 |
< |
lumis.append( (theRun, theLumi) ) |
| 712 |
< |
|
| 713 |
< |
lfns.append(jobFile['lfn']) |
| 711 |
> |
lumisCreated += 1 |
| 712 |
> |
if (not self.limitTotalLumis) or \ |
| 713 |
> |
(lumisCreated <= self.totalNLumis): |
| 714 |
> |
doFile = True |
| 715 |
> |
lumis.append( (theRun, theLumi) ) |
| 716 |
> |
if doFile: |
| 717 |
> |
lfns.append(jobFile['lfn']) |
| 718 |
|
fileString = ','.join(lfns) |
| 719 |
< |
lumiString = compressLumiString(lumis) |
| 719 |
> |
lumiLister = LumiList(lumis = lumis) |
| 720 |
> |
lumiString = lumiLister.getCMSSWString() |
| 721 |
|
list_of_lists.append([fileString, str(-1), str(0), lumiString]) |
| 722 |
|
|
| 723 |
|
jobDestination.append(locations) |
| 724 |
|
jobCount += 1 |
| 711 |
– |
lumisCreated += len(lumis) |
| 725 |
|
common.logger.debug('Job %s will run on %s files and %s lumis ' |
| 726 |
|
% (jobCount, len(lfns), len(lumis) )) |
| 727 |
|
|
| 751 |
|
} |
| 752 |
|
return SplitAlogs |
| 753 |
|
|
| 741 |
– |
|
| 742 |
– |
|
| 743 |
– |
def compressLumiString(lumis): |
| 744 |
– |
""" |
| 745 |
– |
Turn a list of 2-tuples of run/lumi numbers into a list of the format |
| 746 |
– |
R1:L1,R2:L2-R3:L3 which is acceptable to CMSSW LumiBlockRange variable |
| 747 |
– |
""" |
| 748 |
– |
|
| 749 |
– |
lumis.sort() |
| 750 |
– |
parts = [] |
| 751 |
– |
startRange = None |
| 752 |
– |
endRange = None |
| 753 |
– |
|
| 754 |
– |
for lumiBlock in lumis: |
| 755 |
– |
if not startRange: # This is the first one |
| 756 |
– |
startRange = lumiBlock |
| 757 |
– |
endRange = lumiBlock |
| 758 |
– |
elif lumiBlock == endRange: # Same Lumi (different files?) |
| 759 |
– |
pass |
| 760 |
– |
elif lumiBlock[0] == endRange[0] and lumiBlock[1] == endRange[1] + 1: # This is a continuation |
| 761 |
– |
endRange = lumiBlock |
| 762 |
– |
else: # This is the start of a new range |
| 763 |
– |
part = ':'.join(map(str, startRange)) |
| 764 |
– |
if startRange != endRange: |
| 765 |
– |
part += '-' + ':'.join(map(str, endRange)) |
| 766 |
– |
parts.append(part) |
| 767 |
– |
startRange = lumiBlock |
| 768 |
– |
endRange = lumiBlock |
| 769 |
– |
|
| 770 |
– |
# Put out what's left |
| 771 |
– |
if startRange: |
| 772 |
– |
part = ':'.join(map(str, startRange)) |
| 773 |
– |
if startRange != endRange: |
| 774 |
– |
part += '-' + ':'.join(map(str, endRange)) |
| 775 |
– |
parts.append(part) |
| 776 |
– |
|
| 777 |
– |
output = ','.join(parts) |
| 778 |
– |
return output |
| 779 |
– |
|
| 780 |
– |
|
