ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/writeCfg.py

Comparing COMP/CRAB/python/writeCfg.py (file contents):
Revision 1.7 by ewv, Tue May 6 01:09:58 2008 UTC vs.
Revision 1.14 by ewv, Wed Dec 3 17:51:04 2008 UTC

#	Line 1 | Line 1
1		#!/usr/bin/env python
2
3	<	import sys, getopt, string
3	>	"""
4	>	Re-write config file and optionally convert to python
5	>	"""
6	>
7	>	__revision__ = "$Id$"
8	>	__version__ = "$Revision$"
9	>
10	>	import sys, getopt
11		import imp
12		import os
6	–	import random
13		from random import SystemRandom
8	–	_inst  = SystemRandom()
14
15		from ProdCommon.CMSConfigTools.ConfigAPI.CfgInterface import CfgInterface
16	<	from FWCore.ParameterSet.DictTypes import SortedKeysDict
17	<	from FWCore.ParameterSet.Modules   import Service
13	<	from FWCore.ParameterSet.Types     import *
14	<	from FWCore.ParameterSet.Config    import include
16	>	from FWCore.ParameterSet.Config                       import include
17	>	import FWCore.ParameterSet.Types as CfgTypes
18
19	<	import FWCore.ParameterSet.Types   as CfgTypes
17	<	import FWCore.ParameterSet.Modules as CfgModules
19	>	MyRandom  = SystemRandom()
20
21		class ConfigException(Exception):
22	+	"""
23	+	Exceptions raised by writeCfg
24	+	"""
25	+
26		def __init__(self, msg):
27		Exception.__init__(self, msg)
28		self._msg = msg
#	Line 26 | Line 32 | class ConfigException(Exception):
32		return self._msg
33
34		def main(argv) :
35	<	"""
36	<	writeCfg
35	>	"""
36	>	writeCfg
37	>
38	>	- Read in existing, user supplied cfg or pycfg file
39	>	- Modify job specific parameters based on environment variables
40	>	- Write out modified cfg or pycfg file
41	>
42	>	required parameters: none
43	>
44	>	optional parameters:
45	>	--help             :       help
46	>	--debug            :       debug statements
47	>
48	>	"""
49	>
50	>	# defaults
51	>	inputFileNames  = None
52	>	parentFileNames = None
53	>	debug           = False
54	>	_MAXINT         = 900000000
55	>
56	>	try:
57	>	opts, args = getopt.getopt(argv, "", ["debug", "help"])
58	>	except getopt.GetoptError:
59	>	print main.__doc__
60	>	sys.exit(2)
61	>
62	>	try:
63	>	CMSSW  = os.environ['CMSSW_VERSION']
64	>	parts = CMSSW.split('_')
65	>	CMSSW_major = int(parts[1])
66	>	CMSSW_minor = int(parts[2])
67	>	CMSSW_patch = int(parts[3])
68	>	except (KeyError, ValueError):
69	>	msg = "Your environment doesn't specify the CMSSW version or specifies it incorrectly"
70	>	raise ConfigException(msg)
71	>
72	>	# Parse command line options
73	>	for opt, arg in opts :
74	>	if opt  == "--help" :
75	>	print main.__doc__
76	>	sys.exit()
77	>	elif opt == "--debug" :
78	>	debug = True
79
80	<	- Read in existing, user supplied cfg or pycfg file
33	<	- Modify job specific parameters based on environment variables
34	<	- Write out modified cfg or pycfg file
35	<
36	<	required parameters: none
37	<
38	<	optional parameters:
39	<	--help             :       help
40	<	--debug            :       debug statements
41	<
42	<	"""
43	<
44	<	# defaults
45	<	inputFileNames = None
46	<	firstRun       = 0
47	<	sourceSeed     = 0
48	<	vtxSeed        = 0
49	<	g4Seed         = 0
50	<	mixSeed        = 0
51	<	debug          = False
52	<	_MAXINT        = 900000000
53	<	maxSeeds       = 4         # Kludge, maximum # of seeds that any engine takes
54	<
55	<	try:
56	<	opts, args = getopt.getopt(argv, "", ["debug", "help"])
57	<	except getopt.GetoptError:
58	<	print main.__doc__
59	<	sys.exit(2)
60	<
61	<	try:
62	<	CMSSW  = os.environ['CMSSW_VERSION']
63	<	parts = CMSSW.split('_')
64	<	CMSSW_major = int(parts[1])
65	<	CMSSW_minor = int(parts[2])
66	<	CMSSW_patch = int(parts[3])
67	<	except KeyError, ValueError:
68	<	msg = "Your environment doesn't specify the CMSSW version or specifies it incorrectly"
69	<	raise ConfigException(msg)
70	<
71	<	# Parse command line options
72	<	for opt, arg in opts :
73	<	if opt  == "--help" :
74	<	print main.__doc__
75	<	sys.exit()
76	<	elif opt == "--debug" :
77	<	debug = True
78	<
79	<	# Parse remaining parameters
80	<
81	<	try:
82	<	fileName    = args[0];
83	<	outFileName = args[1];
84	<	except IndexError:
85	<	print main.__doc__
86	<	sys.exit()
87	<
88	<	# Optional Parameters
89	<
90	<	maxEvents  = int(os.environ.get('MaxEvents', '0'))
91	<	skipEvents = int(os.environ.get('SkipEvents','0'))
92	<	firstRun   = int(os.environ.get('FirstRun',  '0'))
93	<	nJob       = int(os.environ.get('NJob',      '0'))
94	<
95	<	inputFiles     = os.environ.get('InputFiles','')
96	<	preserveSeeds  = os.environ.get('PreserveSeeds','')
97	<	incrementSeeds = os.environ.get('IncrementSeeds','')
98	<
99	<	# Read Input cfg or python cfg file, FUTURE: Get rid cfg mode
100	<
101	<	if fileName.endswith('py'):
102	<	handle = open(fileName, 'r')
103	<	try:   # Nested form for Python < 2.5
104	<	try:
105	<	cfo = imp.load_source("pycfg", fileName, handle)
106	<	cmsProcess = cfo.process
107	<	except Exception, ex:
108	<	msg = "Your pycfg file is not valid python: %s" % str(ex)
109	<	raise "Error: ",msg
110	<	finally:
111	<	handle.close()
112	<	else:
80	>	# Parse remaining parameters
81		try:
82	<	cfo = include(fileName)
83	<	cmsProcess = cfo
84	<	except Exception, ex:
85	<	msg =  "The cfg file is not valid, %s\n" % str(ex)
86	<	raise "Error: ",msg
87	<	cfg = CfgInterface(cmsProcess)
88	<
89	<	# Set parameters for job
90	<	inModule = cfg.inputSource
91	<	if maxEvents:
92	<	cfg.maxEvents.setMaxEventsInput(maxEvents)
93	<
94	<	if skipEvents:
95	<	inModule.setSkipEvents(skipEvents)
96	<
97	<	if inputFiles:
98	<	inputFiles = inputFiles.replace('\\','')
99	<	inputFiles = inputFiles.replace('"','')
100	<	inputFileNames = inputFiles.split(',')
101	<	inModule.setFileNames(*inputFileNames)
102	<
103	<	# Pythia parameters
104	<	if (firstRun):
105	<	inModule.setFirstRun(firstRun)
106	<
107	<	incrementSeedList = []
108	<	preserveSeedList  = []
109	<
110	<	if incrementSeeds:
111	<	incrementSeedList = incrementSeeds.split(',')
112	<	if preserveSeeds:
113	<	preserveSeedList  = preserveSeeds.split(',')
114	<
115	<	# FUTURE: This function tests the CMSSW version. Can be simplified as we drop support for old versions
116	<	if CMSSW_major < 2 or (CMSSW_major == 2 and CMSSW_minor == 0): # Treatment for seeds, CMSSW < 2_1_x
117	<	if cfg.data.services.has_key('RandomNumberGeneratorService'):
118	<	ranGenerator = cfg.data.services['RandomNumberGeneratorService']
119	<	ranModules   = ranGenerator.moduleSeeds
120	<
121	<	sourceSeed = int(ranGenerator.sourceSeed.value())
122	<	if 'sourceSeed' in preserveSeedList:
123	<	pass
124	<	elif 'sourceSeed' in incrementSeedList:
125	<	ranGenerator.sourceSeed = CfgTypes.untracked(CfgTypes.uint32(sourceSeed+nJob))
126	<	else:
127	<	ranGenerator.sourceSeed = CfgTypes.untracked(CfgTypes.uint32(_inst.randint(1,_MAXINT)))
128	<
129	<	for seed in incrementSeedList:
130	<	curSeed = getattr(ranGenerator.moduleSeeds,seed,None)
131	<	if curSeed:
132	<	curValue = int(curSeed.value())
133	<	setattr(ranGenerator.moduleSeeds,seed,CfgTypes.untracked(CfgTypes.uint32(curValue+nJob)))
134	<	preserveSeedList.append(seed)
135	<
136	<	for seed in ranGenerator.moduleSeeds.parameterNames_():
137	<	if seed not in preserveSeedList:
138	<	curSeed = getattr(ranGenerator.moduleSeeds,seed,None)
139	<	if curSeed:
140	<	curValue = int(curSeed.value())
141	<	setattr(ranGenerator.moduleSeeds,seed,CfgTypes.untracked(CfgTypes.uint32(_inst.randint(1,_MAXINT))))
142	<	else: # Treatment for  seeds, CMSSW 2_1_x and later
143	<	if cfg.data.services.has_key('RandomNumberGeneratorService'):
144	<	from IOMC.RandomEngine.RandomServiceHelper import RandomNumberServiceHelper
145	<
146	<	ranGenerator = cfg.data.services['RandomNumberGeneratorService']
147	<	randSvc = RandomNumberServiceHelper(ranGenerator)
148	<	# sourceSeed is different from the rest, John says will become "theSource"
149	<	if 'sourceSeed' in preserveSeedList:
150	<	pass
151	<	elif 'sourceSeed' in incrementSeedList:
152	<	sourceSeed = int(ranGenerator.sourceSeed.value())
153	<	ranGenerator.sourceSeed = CfgTypes.untracked(CfgTypes.uint32(sourceSeed+nJob))
154	<	else:
155	<	ranGenerator.sourceSeed = CfgTypes.untracked(CfgTypes.uint32(_inst.randint(1,_MAXINT)))
156	<
157	<	# Increment requested seed sets
158	<	for seedName in incrementSeedList:
159	<	curSeeds = randSvc.getNamedSeed(seedName)
160	<	newSeeds = [x+nJob for x in curSeeds]
161	<	randSvc.setNamedSeed(seedName,*newSeeds)
162	<	preserveSeedList.append(seedName)
163	<
164	<	# Randomize remaining seeds
165	<	randSvc.populate(*preserveSeedList)
166	<
167	<	# End version specific code
168	<
169	<	# Write out new config file in one format or the other, FUTURE: Get rid of cfg mode
170	<	outFile = open(outFileName,"w")
171	<	if outFileName.endswith('py'):
172	<	outFile.write("import FWCore.ParameterSet.Config as cms\n")
173	<	outFile.write(cmsProcess.dumpPython())
174	<	if (debug):
175	<	print "writeCfg output:"
176	<	print "import FWCore.ParameterSet.Config as cms"
177	<	print cmsProcess.dumpPython()
178	<	else:
179	<	outFile.write(str(cfg))
180	<	if (debug):
181	<	print "writeCfg output:"
182	<	print str(cfg)
183	<	outFile.close()
82	>	fileName    = args[0]
83	>	outFileName = args[1]
84	>	except IndexError:
85	>	print main.__doc__
86	>	sys.exit()
87	>
88	>	# Optional Parameters
89	>
90	>	maxEvents  = int(os.environ.get('MaxEvents', '0'))
91	>	skipEvents = int(os.environ.get('SkipEvents','0'))
92	>	firstEvent = int(os.environ.get('FirstEvent','0'))
93	>	firstRun   = int(os.environ.get('FirstRun',  '0'))
94	>	nJob       = int(os.environ.get('NJob',      '0'))
95	>
96	>	inputFiles     = os.environ.get('InputFiles','')
97	>	parentFiles    = os.environ.get('ParentFiles','')
98	>	preserveSeeds  = os.environ.get('PreserveSeeds','')
99	>	incrementSeeds = os.environ.get('IncrementSeeds','')
100	>
101	>	# Read Input cfg or python cfg file, FUTURE: Get rid cfg mode
102	>
103	>	if fileName.endswith('py'):
104	>	handle = open(fileName, 'r')
105	>	try:   # Nested form for Python < 2.5
106	>	try:
107	>	print "Importing .py file"
108	>	cfo = imp.load_source("pycfg", fileName, handle)
109	>	cmsProcess = cfo.process
110	>	except Exception, ex:
111	>	msg = "Your pycfg file is not valid python: %s" % str(ex)
112	>	raise ConfigException(msg)
113	>	finally:
114	>	handle.close()
115	>	else:
116	>	try:
117	>	print "Importing .cfg file"
118	>	cfo = include(fileName)
119	>	cmsProcess = cfo
120	>	except Exception, ex:
121	>	msg =  "The cfg file is not valid, %s\n" % str(ex)
122	>	raise ConfigException(msg)
123	>
124	>	cfg = CfgInterface(cmsProcess)
125	>
126	>	# Set parameters for job
127	>	print "Setting parameters"
128	>	inModule = cfg.inputSource
129	>	if maxEvents:
130	>	cfg.maxEvents.setMaxEventsInput(maxEvents)
131	>
132	>	if skipEvents:
133	>	inModule.setSkipEvents(skipEvents)
134	>	if firstEvent:
135	>	cmsProcess.source.firstEvent = CfgTypes.untracked(CfgTypes.uint32(firstEvent))
136	>	if inputFiles:
137	>	inputFiles = inputFiles.replace('\\','')
138	>	inputFiles = inputFiles.replace('"','')
139	>	inputFileNames = inputFiles.split(',')
140	>	inModule.setFileNames(*inputFileNames)
141	>
142	>	# handle parent files if needed
143	>	if parentFiles:
144	>	parentFiles = parentFiles.replace('\\','')
145	>	parentFiles = parentFiles.replace('"','')
146	>	parentFileNames = parentFiles.split(',')
147	>	inModule.setSecondaryFileNames(*parentFileNames)
148	>
149	>	# Pythia parameters
150	>	if (firstRun):
151	>	inModule.setFirstRun(firstRun)
152	>
153	>	incrementSeedList = []
154	>	preserveSeedList  = []
155	>
156	>	if incrementSeeds:
157	>	incrementSeedList = incrementSeeds.split(',')
158	>	if preserveSeeds:
159	>	preserveSeedList  = preserveSeeds.split(',')
160	>
161	>	# FUTURE: This function tests the CMSSW version and presence of old-style seed specification.
162	>	# Reduce when we drop support for old versions
163	>	if cfg.data.services.has_key('RandomNumberGeneratorService'): # There are random #'s to deal with
164	>	print "RandomNumberGeneratorService found, will attempt to change seeds"
165	>	ranGenerator = cfg.data.services['RandomNumberGeneratorService']
166	>	ranModules   = getattr(ranGenerator, "moduleSeeds", None)
167	>	oldSource    = getattr(ranGenerator, "sourceSeed",  None)
168	>	if ranModules != None or oldSource != None:     # Old format present, no matter the CMSSW version
169	>	print "Old-style random number seeds found, will be changed."
170	>	if oldSource != None:
171	>	sourceSeed = int(ranGenerator.sourceSeed.value())
172	>	if ('sourceSeed' in preserveSeedList) or ('theSource' in preserveSeedList):
173	>	pass
174	>	elif ('sourceSeed' in incrementSeedList) or ('theSource' in incrementSeedList):
175	>	ranGenerator.sourceSeed = CfgTypes.untracked(CfgTypes.uint32(sourceSeed+nJob))
176	>	else:
177	>	ranGenerator.sourceSeed = CfgTypes.untracked(CfgTypes.uint32(MyRandom.randint(1, _MAXINT)))
178	>
179	>	for seed in incrementSeedList:
180	>	curSeed = getattr(ranGenerator.moduleSeeds, seed, None)
181	>	if curSeed:
182	>	curValue = int(curSeed.value())
183	>	setattr(ranGenerator.moduleSeeds, seed, CfgTypes.untracked(CfgTypes.uint32(curValue+nJob)))
184	>	preserveSeedList.append(seed)
185	>
186	>	if ranModules != None:
187	>	for seed in ranGenerator.moduleSeeds.parameterNames_():
188	>	if seed not in preserveSeedList:
189	>	curSeed = getattr(ranGenerator.moduleSeeds, seed, None)
190	>	if curSeed:
191	>	curValue = int(curSeed.value())
192	>	setattr(ranGenerator.moduleSeeds, seed,
193	>	CfgTypes.untracked(CfgTypes.uint32(MyRandom.randint(1,_MAXINT))))
194	>	elif CMSSW_major > 2 or (CMSSW_major == 2 and CMSSW_minor >= 1): # Treatment for  seeds, CMSSW 2_1_x and later
195	>	print "New-style random number seeds found, will be changed."
196	>	from IOMC.RandomEngine.RandomServiceHelper import RandomNumberServiceHelper
197	>	randSvc = RandomNumberServiceHelper(ranGenerator)
198	>
199	>	# Increment requested seed sets
200	>	for seedName in incrementSeedList:
201	>	curSeeds = randSvc.getNamedSeed(seedName)
202	>	newSeeds = [x+nJob for x in curSeeds]
203	>	randSvc.setNamedSeed(seedName, *newSeeds)
204	>	preserveSeedList.append(seedName)
205	>
206	>	# Randomize remaining seeds
207	>	randSvc.populate(*preserveSeedList)
208	>	else:
209	>	print "Neither old nor new seed format found!"
210	>
211	>	# End version specific code
212	>
213	>	# Write out new config file in one format or the other, FUTURE: Get rid of cfg mode
214	>	outFile = open(outFileName,"w")
215	>	if outFileName.endswith('py'):
216	>	outFile.write("import FWCore.ParameterSet.Config as cms\n")
217	>	outFile.write(cmsProcess.dumpPython())
218	>	if (debug):
219	>	print "writeCfg output:"
220	>	print "import FWCore.ParameterSet.Config as cms"
221	>	print cmsProcess.dumpPython()
222	>	else:
223	>	outFile.write(cfg.data.dumpConfig())
224	>	if (debug):
225	>	print "writeCfg output:"
226	>	print str(cfg.data.dumpConfig())
227	>	outFile.close()
228
229
230		if __name__ == '__main__' :

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines