| 17 |
|
from random import SystemRandom |
| 18 |
|
|
| 19 |
|
from ProdCommon.CMSConfigTools.ConfigAPI.CfgInterface import CfgInterface |
| 20 |
– |
from FWCore.ParameterSet.Config import include |
| 20 |
|
import FWCore.ParameterSet.Types as CfgTypes |
| 21 |
|
|
| 22 |
|
MyRandom = SystemRandom() |
| 38 |
|
""" |
| 39 |
|
writeCfg |
| 40 |
|
|
| 41 |
< |
- Read in existing, user supplied cfg or pickled pycfg file |
| 41 |
> |
- Read in existing, user supplied pycfg or pickled pycfg file |
| 42 |
|
- Modify job specific parameters based on environment variables and arguments.xml |
| 43 |
< |
- Write out modified cfg or pickled pycfg file |
| 43 |
> |
- Write out pickled pycfg file |
| 44 |
|
|
| 45 |
|
required parameters: none |
| 46 |
|
|
| 101 |
|
firstEvent = -1 |
| 102 |
|
compHEPFirstEvent = 0 |
| 103 |
|
firstRun = 0 |
| 104 |
+ |
# FUTURE: Remove firstRun |
| 105 |
+ |
firstLumi = 0 |
| 106 |
|
|
| 107 |
|
dom = xml.dom.minidom.parse(os.environ['RUNTIME_AREA']+'/arguments.xml') |
| 108 |
|
|
| 116 |
|
firstEvent = int(elem.getAttribute("FirstEvent")) |
| 117 |
|
if elem.getAttribute("FirstRun"): |
| 118 |
|
firstRun = int(elem.getAttribute("FirstRun")) |
| 119 |
+ |
if elem.getAttribute("FirstLumi"): |
| 120 |
+ |
firstLumi = int(elem.getAttribute("FirstLumi")) |
| 121 |
|
|
| 122 |
|
generator = str(elem.getAttribute('Generator')) |
| 123 |
+ |
inputBlocks = str(elem.getAttribute('InputBlocks')) |
| 124 |
|
inputFiles = str(elem.getAttribute('InputFiles')) |
| 125 |
|
parentFiles = str(elem.getAttribute('ParentFiles')) |
| 126 |
|
lumis = str(elem.getAttribute('Lumis')) |
| 127 |
|
|
| 128 |
< |
# Read Input cfg or python cfg file, FUTURE: Get rid cfg mode |
| 128 |
> |
report(inputBlocks,inputFiles,parentFiles,lumis) |
| 129 |
|
|
| 130 |
< |
if fileName.endswith('py'): |
| 131 |
< |
handle = open(fileName, 'r') |
| 132 |
< |
try: # Nested form for Python < 2.5 |
| 133 |
< |
try: |
| 130 |
< |
print "Importing .py file" |
| 131 |
< |
cfo = imp.load_source("pycfg", fileName, handle) |
| 132 |
< |
cmsProcess = cfo.process |
| 133 |
< |
except Exception, ex: |
| 134 |
< |
msg = "Your pycfg file is not valid python: %s" % str(ex) |
| 135 |
< |
raise ConfigException(msg) |
| 136 |
< |
finally: |
| 137 |
< |
handle.close() |
| 138 |
< |
else: |
| 130 |
> |
# Read Input python config file |
| 131 |
> |
|
| 132 |
> |
handle = open(fileName, 'r') |
| 133 |
> |
try: # Nested form for Python < 2.5 |
| 134 |
|
try: |
| 135 |
< |
print "Importing .cfg file" |
| 136 |
< |
cfo = include(fileName) |
| 137 |
< |
cmsProcess = cfo |
| 135 |
> |
print "Importing .py file" |
| 136 |
> |
cfo = imp.load_source("pycfg", fileName, handle) |
| 137 |
> |
cmsProcess = cfo.process |
| 138 |
|
except Exception, ex: |
| 139 |
< |
msg = "The cfg file is not valid, %s\n" % str(ex) |
| 139 |
> |
msg = "Your pycfg file is not valid python: %s" % str(ex) |
| 140 |
|
raise ConfigException(msg) |
| 141 |
+ |
finally: |
| 142 |
+ |
handle.close() |
| 143 |
|
|
| 144 |
|
cfg = CfgInterface(cmsProcess) |
| 145 |
|
|
| 149 |
|
if maxEvents: |
| 150 |
|
cfg.maxEvents.setMaxEventsInput(maxEvents) |
| 151 |
|
|
| 152 |
< |
if skipEvents: |
| 152 |
> |
if skipEvents and inModule.sourceType not in ['EmptySource']: |
| 153 |
|
inModule.setSkipEvents(skipEvents) |
| 154 |
|
|
| 155 |
|
# Set "skip events" for various generators |
| 171 |
|
inModule.setSecondaryFileNames(*parentFileNames) |
| 172 |
|
|
| 173 |
|
if lumis: |
| 174 |
< |
lumiRanges = lumis.split(',') |
| 175 |
< |
inModule.setLumisToProcess(*lumiRanges) |
| 174 |
> |
if CMSSW_major < 3: # FUTURE: Can remove this check |
| 175 |
> |
print "Cannot skip lumis for CMSSW 2_x" |
| 176 |
> |
else: |
| 177 |
> |
lumiRanges = lumis.split(',') |
| 178 |
> |
inModule.setLumisToProcess(*lumiRanges) |
| 179 |
|
|
| 180 |
|
# Pythia parameters |
| 181 |
|
if (firstRun): |
| 182 |
|
inModule.setFirstRun(firstRun) |
| 183 |
+ |
if (firstLumi): |
| 184 |
+ |
inModule.setFirstLumi(firstLumi) |
| 185 |
|
|
| 186 |
< |
incrementSeedList = [] |
| 187 |
< |
preserveSeedList = [] |
| 186 |
< |
|
| 187 |
< |
if incrementSeeds: |
| 188 |
< |
incrementSeedList = incrementSeeds.split(',') |
| 189 |
< |
if preserveSeeds: |
| 190 |
< |
preserveSeedList = preserveSeeds.split(',') |
| 191 |
< |
|
| 192 |
< |
# FUTURE: This function tests the CMSSW version and presence of old-style seed specification. |
| 193 |
< |
# Reduce when we drop support for old versions |
| 194 |
< |
if cfg.data.services.has_key('RandomNumberGeneratorService'): # There are random #'s to deal with |
| 186 |
> |
# Check if there are random #'s to deal with |
| 187 |
> |
if cfg.data.services.has_key('RandomNumberGeneratorService'): |
| 188 |
|
print "RandomNumberGeneratorService found, will attempt to change seeds" |
| 189 |
+ |
from IOMC.RandomEngine.RandomServiceHelper import RandomNumberServiceHelper |
| 190 |
|
ranGenerator = cfg.data.services['RandomNumberGeneratorService'] |
| 191 |
+ |
|
| 192 |
|
ranModules = getattr(ranGenerator, "moduleSeeds", None) |
| 193 |
|
oldSource = getattr(ranGenerator, "sourceSeed", None) |
| 194 |
< |
if ranModules != None or oldSource != None: # Old format present, no matter the CMSSW version |
| 195 |
< |
print "Old-style random number seeds found, will be changed." |
| 196 |
< |
if oldSource != None: |
| 197 |
< |
sourceSeed = int(ranGenerator.sourceSeed.value()) |
| 198 |
< |
if ('sourceSeed' in preserveSeedList) or ('theSource' in preserveSeedList): |
| 204 |
< |
pass |
| 205 |
< |
elif ('sourceSeed' in incrementSeedList) or ('theSource' in incrementSeedList): |
| 206 |
< |
ranGenerator.sourceSeed = CfgTypes.untracked(CfgTypes.uint32(sourceSeed+nJob)) |
| 207 |
< |
else: |
| 208 |
< |
ranGenerator.sourceSeed = CfgTypes.untracked(CfgTypes.uint32(MyRandom.randint(1, _MAXINT))) |
| 209 |
< |
|
| 210 |
< |
for seed in incrementSeedList: |
| 211 |
< |
curSeed = getattr(ranGenerator.moduleSeeds, seed, None) |
| 212 |
< |
if curSeed: |
| 213 |
< |
curValue = int(curSeed.value()) |
| 214 |
< |
setattr(ranGenerator.moduleSeeds, seed, CfgTypes.untracked(CfgTypes.uint32(curValue+nJob))) |
| 215 |
< |
preserveSeedList.append(seed) |
| 216 |
< |
|
| 217 |
< |
if ranModules != None: |
| 218 |
< |
for seed in ranGenerator.moduleSeeds.parameterNames_(): |
| 219 |
< |
if seed not in preserveSeedList: |
| 220 |
< |
curSeed = getattr(ranGenerator.moduleSeeds, seed, None) |
| 221 |
< |
if curSeed: |
| 222 |
< |
curValue = int(curSeed.value()) |
| 223 |
< |
setattr(ranGenerator.moduleSeeds, seed, |
| 224 |
< |
CfgTypes.untracked(CfgTypes.uint32(MyRandom.randint(1,_MAXINT)))) |
| 225 |
< |
elif CMSSW_major > 2 or (CMSSW_major == 2 and CMSSW_minor >= 1): # Treatment for seeds, CMSSW 2_1_x and later |
| 226 |
< |
print "New-style random number seeds found, will be changed." |
| 227 |
< |
from IOMC.RandomEngine.RandomServiceHelper import RandomNumberServiceHelper |
| 228 |
< |
randSvc = RandomNumberServiceHelper(ranGenerator) |
| 229 |
< |
|
| 230 |
< |
# Increment requested seed sets |
| 231 |
< |
for seedName in incrementSeedList: |
| 232 |
< |
curSeeds = randSvc.getNamedSeed(seedName) |
| 233 |
< |
newSeeds = [x+nJob for x in curSeeds] |
| 234 |
< |
randSvc.setNamedSeed(seedName, *newSeeds) |
| 235 |
< |
preserveSeedList.append(seedName) |
| 194 |
> |
if ranModules != None or oldSource != None: |
| 195 |
> |
msg = "Your random number seeds are set in an old,\n" |
| 196 |
> |
msg += "deprecated style. Please change to new style:\n" |
| 197 |
> |
msg += "https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideEDMRandomNumberGeneratorService" |
| 198 |
> |
raise ConfigException(msg) |
| 199 |
|
|
| 200 |
< |
# Randomize remaining seeds |
| 201 |
< |
randSvc.populate(*preserveSeedList) |
| 202 |
< |
else: |
| 203 |
< |
print "Neither old nor new seed format found!" |
| 200 |
> |
randSvc = RandomNumberServiceHelper(ranGenerator) |
| 201 |
> |
|
| 202 |
> |
incrementSeedList = [] |
| 203 |
> |
preserveSeedList = [] |
| 204 |
> |
|
| 205 |
> |
if incrementSeeds: |
| 206 |
> |
incrementSeedList = incrementSeeds.split(',') |
| 207 |
> |
if preserveSeeds: |
| 208 |
> |
preserveSeedList = preserveSeeds.split(',') |
| 209 |
> |
|
| 210 |
> |
# Increment requested seed sets |
| 211 |
> |
for seedName in incrementSeedList: |
| 212 |
> |
curSeeds = randSvc.getNamedSeed(seedName) |
| 213 |
> |
newSeeds = [x+nJob for x in curSeeds] |
| 214 |
> |
randSvc.setNamedSeed(seedName, *newSeeds) |
| 215 |
> |
preserveSeedList.append(seedName) |
| 216 |
|
|
| 217 |
< |
# End version specific code |
| 217 |
> |
# Randomize remaining seeds |
| 218 |
> |
randSvc.populate(*preserveSeedList) |
| 219 |
|
|
| 220 |
< |
# Write out new config file in one format or the other, FUTURE: Get rid of cfg mode |
| 220 |
> |
# Write out new config file |
| 221 |
> |
pklFileName = outFileName + '.pkl' |
| 222 |
|
outFile = open(outFileName,"w") |
| 223 |
< |
if outFileName.endswith('py'): |
| 224 |
< |
outFile.write("import FWCore.ParameterSet.Config as cms\n") |
| 225 |
< |
outFile.write("import pickle\n") |
| 226 |
< |
outFile.write("pickledCfg=\"\"\"%s\"\"\"\n" % pickle.dumps(cmsProcess)) |
| 227 |
< |
outFile.write("process = pickle.loads(pickledCfg)\n") |
| 228 |
< |
if (debug): |
| 229 |
< |
print "writeCfg output (May not be exact):" |
| 230 |
< |
print "import FWCore.ParameterSet.Config as cms" |
| 231 |
< |
print cmsProcess.dumpPython() |
| 232 |
< |
else: |
| 233 |
< |
outFile.write(cfg.data.dumpConfig()) |
| 234 |
< |
if (debug): |
| 235 |
< |
print "writeCfg output:" |
| 236 |
< |
print str(cfg.data.dumpConfig()) |
| 223 |
> |
outFile.write("import FWCore.ParameterSet.Config as cms\n") |
| 224 |
> |
outFile.write("import pickle\n") |
| 225 |
> |
outFile.write("process = pickle.load(open('%s', 'rb'))\n" % pklFileName) |
| 226 |
> |
outFile.close() |
| 227 |
> |
|
| 228 |
> |
pklFile = open(pklFileName,"wb") |
| 229 |
> |
myPickle = pickle.Pickler(pklFile) |
| 230 |
> |
myPickle.dump(cmsProcess) |
| 231 |
> |
pklFile.close() |
| 232 |
> |
|
| 233 |
> |
if (debug): |
| 234 |
> |
print "writeCfg output (May not be exact):" |
| 235 |
> |
print "import FWCore.ParameterSet.Config as cms" |
| 236 |
> |
print cmsProcess.dumpPython() |
| 237 |
> |
|
| 238 |
> |
|
| 239 |
> |
def report( inputBlocks='', inputFiles='', parentFiles='', lumis='' ): |
| 240 |
> |
""" |
| 241 |
> |
Writes the 4 parameters to a file, one parameter per line. |
| 242 |
> |
""" |
| 243 |
> |
outFile = open('inputsReport.txt',"a") |
| 244 |
> |
|
| 245 |
> |
# InputFileList=inputFiles.split(',') |
| 246 |
> |
# parentFilesList= parentFiles.split(',') |
| 247 |
> |
# lumisList= lumis.split(',') |
| 248 |
> |
|
| 249 |
> |
## replacing , with ; otherwise report.py will split it as a new parameter |
| 250 |
> |
txt = '' |
| 251 |
> |
txt += 'inputBlocks='+inputBlocks.replace(',',';')+'\n' |
| 252 |
> |
txt += 'inputFiles='+inputFiles.replace(',',';')+'\n' |
| 253 |
> |
txt += 'parentFiles='+parentFiles.replace(',',';')+'\n' |
| 254 |
> |
txt += 'lumisRange='+lumis.replace(',',';')+'\n' |
| 255 |
> |
outFile.write(str(txt)) |
| 256 |
|
outFile.close() |
| 257 |
+ |
return |
| 258 |
|
|
| 259 |
|
|
| 260 |
|
if __name__ == '__main__' : |
