ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/writeCfg.py

Comparing COMP/CRAB/python/writeCfg.py (file contents):
Revision 1.1 by ewv, Fri Dec 14 17:53:36 2007 UTC vs.
Revision 1.17 by ewv, Mon May 4 19:10:17 2009 UTC

#	Line 1 | Line 1
1		#!/usr/bin/env python
2
3	<	import sys, getopt, string
3	>	"""
4	>	Re-write config file and optionally convert to python
5	>	"""
6	>
7	>	__revision__ = "$Id$"
8	>	__version__ = "$Revision$"
9	>
10	>	import sys, getopt
11		import imp
12		import os
13	+	import pickle
14	+	from random import SystemRandom
15
16		from ProdCommon.CMSConfigTools.ConfigAPI.CfgInterface import CfgInterface
17	<	from FWCore.ParameterSet.DictTypes import SortedKeysDict
18	<	from FWCore.ParameterSet.Modules   import Service
10	<	from FWCore.ParameterSet.Types     import *
11	<	from FWCore.ParameterSet.Config import include
12	<
13	<	import FWCore.ParameterSet.Types   as CfgTypes
14	<	import FWCore.ParameterSet.Modules as CfgModules
15	<
16	<
17	<	def main(argv) :
18	<	"""
19	<	writeCfg
17	>	from FWCore.ParameterSet.Config                       import include
18	>	import FWCore.ParameterSet.Types as CfgTypes
19
20	<	- Read in existing, user supplied cfg or pycfg file
22	<	- Modify job specific parameters based on environment variables
23	<	- Write out modified cfg or pycfg file
24	<
25	<	required parameters: none
26	<
27	<	optional parameters:
28	<	--help             :       help
29	<	--debug            :       debug statements
30	<
31	<	"""
32	<
33	<	# defaults
34	<	maxEvents = 0
35	<	skipEvents = 0
36	<	inputFileNames = None
37	<	debug = False
38	<
39	<	try:
40	<	opts, args = getopt.getopt(argv, "", ["debug", "help","inputFiles=","maxEvents=","skipEvents="])
41	<	except getopt.GetoptError:
42	<	print main.__doc__
43	<	sys.exit(2)
44	<
45	<	# Parse command line parameters
46	<	for opt, arg in opts :
47	<	if opt  == "--help" :
48	<	print main.__doc__
49	<	sys.exit()
50	<	elif opt == "--debug" :
51	<	debug = True
52	<	elif opt == "--maxEvents":
53	<	maxEvents = int(arg)
54	<	elif opt == "--skipEvents":
55	<	skipEvents = int(arg)
56	<	elif opt == "--inputFiles":
57	<	inputFiles = arg
58	<	inputFiles = inputFiles.replace('\\','')
59	<	inputFiles = inputFiles.replace('"','')
60	<	inputFileNames = inputFiles.split(',')
61	<
62	<	# Parse remaining parameters
63	<
64	<	fileName    = args[0];
65	<	outFileName = args[1];
66	<
67	<	# Input cfg or python cfg file
68	<
69	<	if (fileName.endswith('py') or fileName.endswith('pycfg') ):
70	<	handle = open(fileName, 'r')
71	<	try:   # Nested form for Python < 2.5
72	<	try:
73	<	cfo = imp.load_source("pycfg", fileName, handle)
74	<	except Exception, ex:
75	<	msg = "Your pycfg file is not valid python: %s" % str(ex)
76	<	raise "Error: ",msg
77	<	finally:
78	<	handle.close()
79	<	cfg = CfgInterface(cfo.process)
80	<	else:
81	<	try:
82	<	cfo = include(fileName)
83	<	cfg = CfgInterface(cfo)
84	<	except Exception, ex:
85	<	msg =  "The cfg file is not valid, %s\n" % str(ex)
86	<	raise "Error: ",msg
20	>	MyRandom  = SystemRandom()
21
22	<	# Set parameters for job
22	>	class ConfigException(Exception):
23	>	"""
24	>	Exceptions raised by writeCfg
25	>	"""
26	>
27	>	def __init__(self, msg):
28	>	Exception.__init__(self, msg)
29	>	self._msg = msg
30	>	return
31
32	<	inModule = cfg.inputSource
32	>	def __str__(self):
33	>	return self._msg
34
35	<	cfg.maxEvents.setMaxEventsInput(maxEvents)
35	>	def main(argv) :
36	>	"""
37	>	writeCfg
38
39	<	inModule.setSkipEvents(skipEvents)
40	<	if (inputFileNames):
41	<	inModule.setFileNames(*inputFileNames)
39	>	- Read in existing, user supplied cfg or pycfg file
40	>	- Modify job specific parameters based on environment variables
41	>	- Write out modified cfg or pycfg file
42	>
43	>	required parameters: none
44	>
45	>	optional parameters:
46	>	--help             :       help
47	>	--debug            :       debug statements
48	>
49	>	"""
50	>
51	>	# defaults
52	>	inputFileNames  = None
53	>	parentFileNames = None
54	>	debug           = False
55	>	_MAXINT         = 900000000
56
57	<	# Write out new config file
57	>	try:
58	>	opts, args = getopt.getopt(argv, "", ["debug", "help"])
59	>	except getopt.GetoptError:
60	>	print main.__doc__
61	>	sys.exit(2)
62
63	<	outFile = open(outFileName,"w")
64	<	outFile.write("import FWCore.ParameterSet.Config as cms\n")
65	<	outFile.write(cfo.dumpPython())
66	<	outFile.close()
63	>	try:
64	>	CMSSW  = os.environ['CMSSW_VERSION']
65	>	parts = CMSSW.split('_')
66	>	CMSSW_major = int(parts[1])
67	>	CMSSW_minor = int(parts[2])
68	>	CMSSW_patch = int(parts[3])
69	>	except (KeyError, ValueError):
70	>	msg = "Your environment doesn't specify the CMSSW version or specifies it incorrectly"
71	>	raise ConfigException(msg)
72	>
73	>	# Parse command line options
74	>	for opt, arg in opts :
75	>	if opt  == "--help" :
76	>	print main.__doc__
77	>	sys.exit()
78	>	elif opt == "--debug" :
79	>	debug = True
80
81	<	if (debug):
82	<	print "writeCfg output:"
83	<	print "import FWCore.ParameterSet.Config as cms"
84	<	print cfo.dumpPython()
81	>	# Parse remaining parameters
82	>	try:
83	>	fileName    = args[0]
84	>	outFileName = args[1]
85	>	except IndexError:
86	>	print main.__doc__
87	>	sys.exit()
88	>
89	>	# Optional Parameters
90	>
91	>	maxEvents  = int(os.environ.get('MaxEvents', '0'))
92	>	skipEvents = int(os.environ.get('SkipEvents','0'))
93	>	firstEvent = int(os.environ.get('FirstEvent','-1'))
94	>	compHEPFirstEvent = int(os.environ.get('CompHEPFirstEvent','0'))
95	>	firstRun   = int(os.environ.get('FirstRun',  '0'))
96	>	nJob       = int(os.environ.get('NJob',      '0'))
97	>
98	>	inputFiles     = os.environ.get('InputFiles','')
99	>	parentFiles    = os.environ.get('ParentFiles','')
100	>	preserveSeeds  = os.environ.get('PreserveSeeds','')
101	>	incrementSeeds = os.environ.get('IncrementSeeds','')
102	>
103	>	# Read Input cfg or python cfg file, FUTURE: Get rid cfg mode
104	>
105	>	if fileName.endswith('py'):
106	>	handle = open(fileName, 'r')
107	>	try:   # Nested form for Python < 2.5
108	>	try:
109	>	print "Importing .py file"
110	>	cfo = imp.load_source("pycfg", fileName, handle)
111	>	cmsProcess = cfo.process
112	>	except Exception, ex:
113	>	msg = "Your pycfg file is not valid python: %s" % str(ex)
114	>	raise ConfigException(msg)
115	>	finally:
116	>	handle.close()
117	>	else:
118	>	try:
119	>	print "Importing .cfg file"
120	>	cfo = include(fileName)
121	>	cmsProcess = cfo
122	>	except Exception, ex:
123	>	msg =  "The cfg file is not valid, %s\n" % str(ex)
124	>	raise ConfigException(msg)
125	>
126	>	cfg = CfgInterface(cmsProcess)
127	>
128	>	# Set parameters for job
129	>	print "Setting parameters"
130	>	inModule = cfg.inputSource
131	>	if maxEvents:
132	>	cfg.maxEvents.setMaxEventsInput(maxEvents)
133	>
134	>	if skipEvents:
135	>	inModule.setSkipEvents(skipEvents)
136	>	if firstEvent != -1:
137	>	cmsProcess.source.firstEvent = CfgTypes.untracked(CfgTypes.uint32(firstEvent))
138	>	if compHEPFirstEvent:
139	>	cmsProcess.source.CompHEPFirstEvent = CfgTypes.int32(compHEPFirstEvent)
140	>	if inputFiles:
141	>	inputFiles = inputFiles.replace('\\','')
142	>	inputFiles = inputFiles.replace('"','')
143	>	inputFileNames = inputFiles.split(',')
144	>	inModule.setFileNames(*inputFileNames)
145	>
146	>	# handle parent files if needed
147	>	if parentFiles:
148	>	parentFiles = parentFiles.replace('\\','')
149	>	parentFiles = parentFiles.replace('"','')
150	>	parentFileNames = parentFiles.split(',')
151	>	inModule.setSecondaryFileNames(*parentFileNames)
152	>
153	>	# Pythia parameters
154	>	if (firstRun):
155	>	inModule.setFirstRun(firstRun)
156	>
157	>	incrementSeedList = []
158	>	preserveSeedList  = []
159	>
160	>	if incrementSeeds:
161	>	incrementSeedList = incrementSeeds.split(',')
162	>	if preserveSeeds:
163	>	preserveSeedList  = preserveSeeds.split(',')
164	>
165	>	# FUTURE: This function tests the CMSSW version and presence of old-style seed specification.
166	>	# Reduce when we drop support for old versions
167	>	if cfg.data.services.has_key('RandomNumberGeneratorService'): # There are random #'s to deal with
168	>	print "RandomNumberGeneratorService found, will attempt to change seeds"
169	>	ranGenerator = cfg.data.services['RandomNumberGeneratorService']
170	>	ranModules   = getattr(ranGenerator, "moduleSeeds", None)
171	>	oldSource    = getattr(ranGenerator, "sourceSeed",  None)
172	>	if ranModules != None or oldSource != None:     # Old format present, no matter the CMSSW version
173	>	print "Old-style random number seeds found, will be changed."
174	>	if oldSource != None:
175	>	sourceSeed = int(ranGenerator.sourceSeed.value())
176	>	if ('sourceSeed' in preserveSeedList) or ('theSource' in preserveSeedList):
177	>	pass
178	>	elif ('sourceSeed' in incrementSeedList) or ('theSource' in incrementSeedList):
179	>	ranGenerator.sourceSeed = CfgTypes.untracked(CfgTypes.uint32(sourceSeed+nJob))
180	>	else:
181	>	ranGenerator.sourceSeed = CfgTypes.untracked(CfgTypes.uint32(MyRandom.randint(1, _MAXINT)))
182	>
183	>	for seed in incrementSeedList:
184	>	curSeed = getattr(ranGenerator.moduleSeeds, seed, None)
185	>	if curSeed:
186	>	curValue = int(curSeed.value())
187	>	setattr(ranGenerator.moduleSeeds, seed, CfgTypes.untracked(CfgTypes.uint32(curValue+nJob)))
188	>	preserveSeedList.append(seed)
189	>
190	>	if ranModules != None:
191	>	for seed in ranGenerator.moduleSeeds.parameterNames_():
192	>	if seed not in preserveSeedList:
193	>	curSeed = getattr(ranGenerator.moduleSeeds, seed, None)
194	>	if curSeed:
195	>	curValue = int(curSeed.value())
196	>	setattr(ranGenerator.moduleSeeds, seed,
197	>	CfgTypes.untracked(CfgTypes.uint32(MyRandom.randint(1,_MAXINT))))
198	>	elif CMSSW_major > 2 or (CMSSW_major == 2 and CMSSW_minor >= 1): # Treatment for  seeds, CMSSW 2_1_x and later
199	>	print "New-style random number seeds found, will be changed."
200	>	from IOMC.RandomEngine.RandomServiceHelper import RandomNumberServiceHelper
201	>	randSvc = RandomNumberServiceHelper(ranGenerator)
202	>
203	>	# Increment requested seed sets
204	>	for seedName in incrementSeedList:
205	>	curSeeds = randSvc.getNamedSeed(seedName)
206	>	newSeeds = [x+nJob for x in curSeeds]
207	>	randSvc.setNamedSeed(seedName, *newSeeds)
208	>	preserveSeedList.append(seedName)
209	>
210	>	# Randomize remaining seeds
211	>	randSvc.populate(*preserveSeedList)
212	>	else:
213	>	print "Neither old nor new seed format found!"
214	>
215	>	# End version specific code
216	>
217	>	# Write out new config file in one format or the other, FUTURE: Get rid of cfg mode
218	>	outFile = open(outFileName,"w")
219	>	if outFileName.endswith('py'):
220	>	outFile.write("import FWCore.ParameterSet.Config as cms\n")
221	>	outFile.write("import pickle\n")
222	>	outFile.write("pickledCfg=\"\"\"%s\"\"\"\n" % pickle.dumps(cmsProcess))
223	>	outFile.write("process = pickle.loads(pickledCfg)\n")
224	>	if (debug):
225	>	print "writeCfg output (May not be exact):"
226	>	print "import FWCore.ParameterSet.Config as cms"
227	>	print cmsProcess.dumpPython()
228	>	else:
229	>	outFile.write(cfg.data.dumpConfig())
230	>	if (debug):
231	>	print "writeCfg output:"
232	>	print str(cfg.data.dumpConfig())
233	>	outFile.close()
234
235
236		if __name__ == '__main__' :
237		exit_status = main(sys.argv[1:])
238		sys.exit(exit_status)
114	–

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines