| 7 |
|
__revision__ = "$Id$" |
| 8 |
|
__version__ = "$Revision$" |
| 9 |
|
|
| 10 |
< |
import sys, getopt |
| 10 |
> |
import getopt |
| 11 |
|
import imp |
| 12 |
|
import os |
| 13 |
+ |
import pickle |
| 14 |
+ |
import sys |
| 15 |
+ |
import xml.dom.minidom |
| 16 |
+ |
|
| 17 |
|
from random import SystemRandom |
| 18 |
|
|
| 19 |
|
from ProdCommon.CMSConfigTools.ConfigAPI.CfgInterface import CfgInterface |
| 16 |
– |
from FWCore.ParameterSet.Config import include |
| 20 |
|
import FWCore.ParameterSet.Types as CfgTypes |
| 21 |
|
|
| 22 |
|
MyRandom = SystemRandom() |
| 38 |
|
""" |
| 39 |
|
writeCfg |
| 40 |
|
|
| 41 |
< |
- Read in existing, user supplied cfg or pycfg file |
| 42 |
< |
- Modify job specific parameters based on environment variables |
| 43 |
< |
- Write out modified cfg or pycfg file |
| 41 |
> |
- Read in existing, user supplied pycfg or pickled pycfg file |
| 42 |
> |
- Modify job specific parameters based on environment variables and arguments.xml |
| 43 |
> |
- Write out pickled pycfg file |
| 44 |
|
|
| 45 |
|
required parameters: none |
| 46 |
|
|
| 88 |
|
print main.__doc__ |
| 89 |
|
sys.exit() |
| 90 |
|
|
| 91 |
< |
# Optional Parameters |
| 91 |
> |
# Read in Environment, XML and get optional Parameters |
| 92 |
|
|
| 90 |
– |
maxEvents = int(os.environ.get('MaxEvents', '0')) |
| 91 |
– |
skipEvents = int(os.environ.get('SkipEvents','0')) |
| 92 |
– |
firstRun = int(os.environ.get('FirstRun', '0')) |
| 93 |
|
nJob = int(os.environ.get('NJob', '0')) |
| 94 |
– |
|
| 95 |
– |
inputFiles = os.environ.get('InputFiles','') |
| 96 |
– |
parentFiles = os.environ.get('ParentFiles','') |
| 94 |
|
preserveSeeds = os.environ.get('PreserveSeeds','') |
| 95 |
|
incrementSeeds = os.environ.get('IncrementSeeds','') |
| 96 |
|
|
| 97 |
< |
# Read Input cfg or python cfg file, FUTURE: Get rid cfg mode |
| 97 |
> |
# Defaults |
| 98 |
> |
|
| 99 |
> |
maxEvents = 0 |
| 100 |
> |
skipEvents = 0 |
| 101 |
> |
firstEvent = -1 |
| 102 |
> |
compHEPFirstEvent = 0 |
| 103 |
> |
firstRun = 0 |
| 104 |
> |
# FUTURE: Remove firstRun |
| 105 |
> |
firstLumi = 0 |
| 106 |
> |
|
| 107 |
> |
dom = xml.dom.minidom.parse(os.environ['RUNTIME_AREA']+'/arguments.xml') |
| 108 |
> |
|
| 109 |
> |
for elem in dom.getElementsByTagName("Job"): |
| 110 |
> |
if nJob == int(elem.getAttribute("JobID")): |
| 111 |
> |
if elem.getAttribute("MaxEvents"): |
| 112 |
> |
maxEvents = int(elem.getAttribute("MaxEvents")) |
| 113 |
> |
if elem.getAttribute("SkipEvents"): |
| 114 |
> |
skipEvents = int(elem.getAttribute("SkipEvents")) |
| 115 |
> |
if elem.getAttribute("FirstEvent"): |
| 116 |
> |
firstEvent = int(elem.getAttribute("FirstEvent")) |
| 117 |
> |
if elem.getAttribute("FirstRun"): |
| 118 |
> |
firstRun = int(elem.getAttribute("FirstRun")) |
| 119 |
> |
if elem.getAttribute("FirstLumi"): |
| 120 |
> |
firstLumi = int(elem.getAttribute("FirstLumi")) |
| 121 |
> |
|
| 122 |
> |
generator = str(elem.getAttribute('Generator')) |
| 123 |
> |
inputBlocks = str(elem.getAttribute('InputBlocks')) |
| 124 |
> |
inputFiles = str(elem.getAttribute('InputFiles')) |
| 125 |
> |
parentFiles = str(elem.getAttribute('ParentFiles')) |
| 126 |
> |
lumis = str(elem.getAttribute('Lumis')) |
| 127 |
|
|
| 128 |
< |
if fileName.endswith('py'): |
| 129 |
< |
handle = open(fileName, 'r') |
| 130 |
< |
try: # Nested form for Python < 2.5 |
| 131 |
< |
try: |
| 132 |
< |
print "Importing .py file" |
| 133 |
< |
cfo = imp.load_source("pycfg", fileName, handle) |
| 108 |
< |
cmsProcess = cfo.process |
| 109 |
< |
except Exception, ex: |
| 110 |
< |
msg = "Your pycfg file is not valid python: %s" % str(ex) |
| 111 |
< |
raise ConfigException(msg) |
| 112 |
< |
finally: |
| 113 |
< |
handle.close() |
| 114 |
< |
else: |
| 128 |
> |
report(inputBlocks,inputFiles,parentFiles,lumis) |
| 129 |
> |
|
| 130 |
> |
# Read Input python config file |
| 131 |
> |
|
| 132 |
> |
handle = open(fileName, 'r') |
| 133 |
> |
try: # Nested form for Python < 2.5 |
| 134 |
|
try: |
| 135 |
< |
print "Importing .cfg file" |
| 136 |
< |
cfo = include(fileName) |
| 137 |
< |
cmsProcess = cfo |
| 135 |
> |
print "Importing .py file" |
| 136 |
> |
cfo = imp.load_source("pycfg", fileName, handle) |
| 137 |
> |
cmsProcess = cfo.process |
| 138 |
|
except Exception, ex: |
| 139 |
< |
msg = "The cfg file is not valid, %s\n" % str(ex) |
| 139 |
> |
msg = "Your pycfg file is not valid python: %s" % str(ex) |
| 140 |
|
raise ConfigException(msg) |
| 141 |
+ |
finally: |
| 142 |
+ |
handle.close() |
| 143 |
|
|
| 144 |
|
cfg = CfgInterface(cmsProcess) |
| 145 |
|
|
| 149 |
|
if maxEvents: |
| 150 |
|
cfg.maxEvents.setMaxEventsInput(maxEvents) |
| 151 |
|
|
| 152 |
< |
if skipEvents: |
| 152 |
> |
if skipEvents and inModule.sourceType not in ['EmptySource']: |
| 153 |
|
inModule.setSkipEvents(skipEvents) |
| 154 |
|
|
| 155 |
+ |
# Set "skip events" for various generators |
| 156 |
+ |
if generator == 'comphep': |
| 157 |
+ |
cmsProcess.source.CompHEPFirstEvent = CfgTypes.int32(firstEvent) |
| 158 |
+ |
elif generator == 'lhe': |
| 159 |
+ |
cmsProcess.source.skipEvents = CfgTypes.untracked(CfgTypes.uint32(firstEvent)) |
| 160 |
+ |
cmsProcess.source.firstEvent = CfgTypes.untracked(CfgTypes.uint32(firstEvent+1)) |
| 161 |
+ |
elif firstEvent != -1: # (Old? Madgraph) |
| 162 |
+ |
cmsProcess.source.firstEvent = CfgTypes.untracked(CfgTypes.uint32(firstEvent)) |
| 163 |
+ |
|
| 164 |
|
if inputFiles: |
| 135 |
– |
inputFiles = inputFiles.replace('\\','') |
| 136 |
– |
inputFiles = inputFiles.replace('"','') |
| 165 |
|
inputFileNames = inputFiles.split(',') |
| 166 |
|
inModule.setFileNames(*inputFileNames) |
| 167 |
|
|
| 168 |
|
# handle parent files if needed |
| 169 |
|
if parentFiles: |
| 142 |
– |
parentFiles = parentFiles.replace('\\','') |
| 143 |
– |
parentFiles = parentFiles.replace('"','') |
| 170 |
|
parentFileNames = parentFiles.split(',') |
| 171 |
|
inModule.setSecondaryFileNames(*parentFileNames) |
| 172 |
|
|
| 173 |
+ |
if lumis: |
| 174 |
+ |
if CMSSW_major < 3: # FUTURE: Can remove this check |
| 175 |
+ |
print "Cannot skip lumis for CMSSW 2_x" |
| 176 |
+ |
else: |
| 177 |
+ |
lumiRanges = lumis.split(',') |
| 178 |
+ |
inModule.setLumisToProcess(*lumiRanges) |
| 179 |
+ |
|
| 180 |
|
# Pythia parameters |
| 181 |
|
if (firstRun): |
| 182 |
|
inModule.setFirstRun(firstRun) |
| 183 |
+ |
if (firstLumi): |
| 184 |
+ |
inModule.setFirstLumi(firstLumi) |
| 185 |
|
|
| 186 |
< |
incrementSeedList = [] |
| 187 |
< |
preserveSeedList = [] |
| 153 |
< |
|
| 154 |
< |
if incrementSeeds: |
| 155 |
< |
incrementSeedList = incrementSeeds.split(',') |
| 156 |
< |
if preserveSeeds: |
| 157 |
< |
preserveSeedList = preserveSeeds.split(',') |
| 158 |
< |
|
| 159 |
< |
# FUTURE: This function tests the CMSSW version and presence of old-style seed specification. |
| 160 |
< |
# Reduce when we drop support for old versions |
| 161 |
< |
if cfg.data.services.has_key('RandomNumberGeneratorService'): # There are random #'s to deal with |
| 186 |
> |
# Check if there are random #'s to deal with |
| 187 |
> |
if cfg.data.services.has_key('RandomNumberGeneratorService'): |
| 188 |
|
print "RandomNumberGeneratorService found, will attempt to change seeds" |
| 189 |
+ |
from IOMC.RandomEngine.RandomServiceHelper import RandomNumberServiceHelper |
| 190 |
|
ranGenerator = cfg.data.services['RandomNumberGeneratorService'] |
| 191 |
+ |
|
| 192 |
|
ranModules = getattr(ranGenerator, "moduleSeeds", None) |
| 193 |
|
oldSource = getattr(ranGenerator, "sourceSeed", None) |
| 194 |
< |
if ranModules != None or oldSource != None: # Old format present, no matter the CMSSW version |
| 195 |
< |
print "Old-style random number seeds found, will be changed." |
| 196 |
< |
if oldSource != None: |
| 197 |
< |
sourceSeed = int(ranGenerator.sourceSeed.value()) |
| 198 |
< |
if ('sourceSeed' in preserveSeedList) or ('theSource' in preserveSeedList): |
| 171 |
< |
pass |
| 172 |
< |
elif ('sourceSeed' in incrementSeedList) or ('theSource' in incrementSeedList): |
| 173 |
< |
ranGenerator.sourceSeed = CfgTypes.untracked(CfgTypes.uint32(sourceSeed+nJob)) |
| 174 |
< |
else: |
| 175 |
< |
ranGenerator.sourceSeed = CfgTypes.untracked(CfgTypes.uint32(MyRandom.randint(1, _MAXINT))) |
| 176 |
< |
|
| 177 |
< |
for seed in incrementSeedList: |
| 178 |
< |
curSeed = getattr(ranGenerator.moduleSeeds, seed, None) |
| 179 |
< |
if curSeed: |
| 180 |
< |
curValue = int(curSeed.value()) |
| 181 |
< |
setattr(ranGenerator.moduleSeeds, seed, CfgTypes.untracked(CfgTypes.uint32(curValue+nJob))) |
| 182 |
< |
preserveSeedList.append(seed) |
| 183 |
< |
|
| 184 |
< |
if ranModules != None: |
| 185 |
< |
for seed in ranGenerator.moduleSeeds.parameterNames_(): |
| 186 |
< |
if seed not in preserveSeedList: |
| 187 |
< |
curSeed = getattr(ranGenerator.moduleSeeds, seed, None) |
| 188 |
< |
if curSeed: |
| 189 |
< |
curValue = int(curSeed.value()) |
| 190 |
< |
setattr(ranGenerator.moduleSeeds, seed, |
| 191 |
< |
CfgTypes.untracked(CfgTypes.uint32(MyRandom.randint(1,_MAXINT)))) |
| 192 |
< |
elif CMSSW_major > 2 or (CMSSW_major == 2 and CMSSW_minor >= 1): # Treatment for seeds, CMSSW 2_1_x and later |
| 193 |
< |
print "New-style random number seeds found, will be changed." |
| 194 |
< |
from IOMC.RandomEngine.RandomServiceHelper import RandomNumberServiceHelper |
| 195 |
< |
randSvc = RandomNumberServiceHelper(ranGenerator) |
| 196 |
< |
|
| 197 |
< |
# Increment requested seed sets |
| 198 |
< |
for seedName in incrementSeedList: |
| 199 |
< |
curSeeds = randSvc.getNamedSeed(seedName) |
| 200 |
< |
newSeeds = [x+nJob for x in curSeeds] |
| 201 |
< |
randSvc.setNamedSeed(seedName, *newSeeds) |
| 202 |
< |
preserveSeedList.append(seedName) |
| 194 |
> |
if ranModules != None or oldSource != None: |
| 195 |
> |
msg = "Your random number seeds are set in an old,\n" |
| 196 |
> |
msg += "deprecated style. Please change to new style:\n" |
| 197 |
> |
msg += "https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideEDMRandomNumberGeneratorService" |
| 198 |
> |
raise ConfigException(msg) |
| 199 |
|
|
| 200 |
< |
# Randomize remaining seeds |
| 201 |
< |
randSvc.populate(*preserveSeedList) |
| 202 |
< |
else: |
| 203 |
< |
print "Neither old nor new seed format found!" |
| 200 |
> |
randSvc = RandomNumberServiceHelper(ranGenerator) |
| 201 |
> |
|
| 202 |
> |
incrementSeedList = [] |
| 203 |
> |
preserveSeedList = [] |
| 204 |
> |
|
| 205 |
> |
if incrementSeeds: |
| 206 |
> |
incrementSeedList = incrementSeeds.split(',') |
| 207 |
> |
if preserveSeeds: |
| 208 |
> |
preserveSeedList = preserveSeeds.split(',') |
| 209 |
> |
|
| 210 |
> |
# Increment requested seed sets |
| 211 |
> |
for seedName in incrementSeedList: |
| 212 |
> |
curSeeds = randSvc.getNamedSeed(seedName) |
| 213 |
> |
newSeeds = [x+nJob for x in curSeeds] |
| 214 |
> |
randSvc.setNamedSeed(seedName, *newSeeds) |
| 215 |
> |
preserveSeedList.append(seedName) |
| 216 |
|
|
| 217 |
< |
# End version specific code |
| 217 |
> |
# Randomize remaining seeds |
| 218 |
> |
randSvc.populate(*preserveSeedList) |
| 219 |
|
|
| 220 |
< |
# Write out new config file in one format or the other, FUTURE: Get rid of cfg mode |
| 220 |
> |
# Write out new config file |
| 221 |
> |
pklFileName = outFileName + '.pkl' |
| 222 |
|
outFile = open(outFileName,"w") |
| 223 |
< |
if outFileName.endswith('py'): |
| 224 |
< |
outFile.write("import FWCore.ParameterSet.Config as cms\n") |
| 225 |
< |
outFile.write(cmsProcess.dumpPython()) |
| 226 |
< |
if (debug): |
| 227 |
< |
print "writeCfg output:" |
| 228 |
< |
print "import FWCore.ParameterSet.Config as cms" |
| 229 |
< |
print cmsProcess.dumpPython() |
| 230 |
< |
else: |
| 231 |
< |
outFile.write(cfg.data.dumpConfig()) |
| 232 |
< |
if (debug): |
| 233 |
< |
print "writeCfg output:" |
| 234 |
< |
print str(cfg.data.dumpConfig()) |
| 223 |
> |
outFile.write("import FWCore.ParameterSet.Config as cms\n") |
| 224 |
> |
outFile.write("import pickle\n") |
| 225 |
> |
outFile.write("process = pickle.load(open('%s', 'rb'))\n" % pklFileName) |
| 226 |
> |
outFile.close() |
| 227 |
> |
|
| 228 |
> |
pklFile = open(pklFileName,"wb") |
| 229 |
> |
myPickle = pickle.Pickler(pklFile) |
| 230 |
> |
myPickle.dump(cmsProcess) |
| 231 |
> |
pklFile.close() |
| 232 |
> |
|
| 233 |
> |
if (debug): |
| 234 |
> |
print "writeCfg output (May not be exact):" |
| 235 |
> |
print "import FWCore.ParameterSet.Config as cms" |
| 236 |
> |
print cmsProcess.dumpPython() |
| 237 |
> |
|
| 238 |
> |
|
| 239 |
> |
def report( inputBlocks='', inputFiles='', parentFiles='', lumis='' ): |
| 240 |
> |
""" |
| 241 |
> |
Writes the 4 parameters to a file, one parameter per line. |
| 242 |
> |
""" |
| 243 |
> |
outFile = open('inputsReport.txt',"a") |
| 244 |
> |
|
| 245 |
> |
# InputFileList=inputFiles.split(',') |
| 246 |
> |
# parentFilesList= parentFiles.split(',') |
| 247 |
> |
# lumisList= lumis.split(',') |
| 248 |
> |
|
| 249 |
> |
## replacing , with ; otherwise report.py will split it as a new parameter |
| 250 |
> |
txt = '' |
| 251 |
> |
txt += 'inputBlocks='+inputBlocks.replace(',',';')+'\n' |
| 252 |
> |
txt += 'inputFiles='+inputFiles.replace(',',';')+'\n' |
| 253 |
> |
txt += 'parentFiles='+parentFiles.replace(',',';')+'\n' |
| 254 |
> |
txt += 'lumisRange='+lumis.replace(',',';')+'\n' |
| 255 |
> |
print txt |
| 256 |
> |
outFile.write(str(txt)) |
| 257 |
|
outFile.close() |
| 258 |
+ |
return |
| 259 |
|
|
| 260 |
|
|
| 261 |
|
if __name__ == '__main__' : |
