ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/writeCfg.py

Comparing COMP/CRAB/python/writeCfg.py (file contents):
Revision 1.6 by ewv, Wed Apr 30 19:41:29 2008 UTC vs.
Revision 1.26 by ewv, Mon Dec 14 22:33:54 2009 UTC

#	Line 1 | Line 1
1		#!/usr/bin/env python
2
3	<	import sys, getopt, string
3	>	"""
4	>	Re-write config file and optionally convert to python
5	>	"""
6	>
7	>	__revision__ = "$Id$"
8	>	__version__ = "$Revision$"
9	>
10	>	import getopt
11		import imp
12		import os
13	<	import random
13	>	import pickle
14	>	import sys
15	>	import xml.dom.minidom
16	>
17		from random import SystemRandom
8	–	_inst  = SystemRandom()
18
19		from ProdCommon.CMSConfigTools.ConfigAPI.CfgInterface import CfgInterface
20	<	from FWCore.ParameterSet.DictTypes import SortedKeysDict
21	<	from FWCore.ParameterSet.Modules   import Service
13	<	from FWCore.ParameterSet.Types     import *
14	<	from FWCore.ParameterSet.Config    import include
20	>	from FWCore.ParameterSet.Config                       import include
21	>	import FWCore.ParameterSet.Types as CfgTypes
22
23	<	import FWCore.ParameterSet.Types   as CfgTypes
17	<	import FWCore.ParameterSet.Modules as CfgModules
23	>	MyRandom  = SystemRandom()
24
25		class ConfigException(Exception):
26	+	"""
27	+	Exceptions raised by writeCfg
28	+	"""
29	+
30		def __init__(self, msg):
31		Exception.__init__(self, msg)
32		self._msg = msg
#	Line 26 | Line 36 | class ConfigException(Exception):
36		return self._msg
37
38		def main(argv) :
39	<	"""
40	<	writeCfg
39	>	"""
40	>	writeCfg
41	>
42	>	- Read in existing, user supplied pycfg or pickled pycfg file
43	>	- Modify job specific parameters based on environment variables and arguments.xml
44	>	- Write out pickled pycfg file
45	>
46	>	required parameters: none
47	>
48	>	optional parameters:
49	>	--help             :       help
50	>	--debug            :       debug statements
51	>
52	>	"""
53	>
54	>	# defaults
55	>	inputFileNames  = None
56	>	parentFileNames = None
57	>	debug           = False
58	>	_MAXINT         = 900000000
59	>
60	>	try:
61	>	opts, args = getopt.getopt(argv, "", ["debug", "help"])
62	>	except getopt.GetoptError:
63	>	print main.__doc__
64	>	sys.exit(2)
65	>
66	>	try:
67	>	CMSSW  = os.environ['CMSSW_VERSION']
68	>	parts = CMSSW.split('_')
69	>	CMSSW_major = int(parts[1])
70	>	CMSSW_minor = int(parts[2])
71	>	CMSSW_patch = int(parts[3])
72	>	except (KeyError, ValueError):
73	>	msg = "Your environment doesn't specify the CMSSW version or specifies it incorrectly"
74	>	raise ConfigException(msg)
75	>
76	>	# Parse command line options
77	>	for opt, arg in opts :
78	>	if opt  == "--help" :
79	>	print main.__doc__
80	>	sys.exit()
81	>	elif opt == "--debug" :
82	>	debug = True
83
84	<	- Read in existing, user supplied cfg or pycfg file
85	<	- Modify job specific parameters based on environment variables
86	<	- Write out modified cfg or pycfg file
87	<
88	<	required parameters: none
89	<
90	<	optional parameters:
91	<	--help             :       help
92	<	--debug            :       debug statements
93	<
94	<	"""
95	<
96	<	# defaults
97	<	inputFileNames = None
98	<	firstRun       = 0
99	<	sourceSeed     = 0
100	<	vtxSeed        = 0
101	<	g4Seed         = 0
102	<	mixSeed        = 0
103	<	debug          = False
104	<
105	<	try:
106	<	opts, args = getopt.getopt(argv, "", ["debug", "help"])
107	<	except getopt.GetoptError:
108	<	print main.__doc__
109	<	sys.exit(2)
110	<
111	<	try:
112	<	CMSSW  = os.environ['CMSSW_VERSION']
113	<	parts = CMSSW.split('_')
114	<	CMSSW_major = int(parts[1])
115	<	CMSSW_minor = int(parts[2])
116	<	CMSSW_patch = int(parts[3])
117	<	except KeyError, ValueError:
118	<	msg = "Your environment doesn't specify the CMSSW version or specifies it incorrectly"
119	<	raise ConfigException(msg)
120	<
121	<	# Parse command line options
122	<	for opt, arg in opts :
123	<	if opt  == "--help" :
124	<	print main.__doc__
125	<	sys.exit()
126	<	elif opt == "--debug" :
75	<	debug = True
76	<
77	<	# Parse remaining parameters
78	<
79	<	try:
80	<	fileName    = args[0];
81	<	outFileName = args[1];
82	<	except IndexError:
83	<	print main.__doc__
84	<	sys.exit()
85	<
86	<	# Optional Parameters
87	<
88	<	maxEvents  = int(os.environ.get('MaxEvents','0'))
89	<	skipEvents = int(os.environ.get('SkipEvents','0'))
90	<	inputFiles = os.environ.get('InputFiles','')
91	<	firstRun   = int(os.environ.get('FirstRun','0'))
92	<	nJob       = int(os.environ.get('NJob','0'))
93	<	preserveSeeds = os.environ.get('PreserveSeeds','')
94	<	incrementSeeds = os.environ.get('IncrementSeeds','')
84	>	# Parse remaining parameters
85	>	try:
86	>	fileName    = args[0]
87	>	outFileName = args[1]
88	>	except IndexError:
89	>	print main.__doc__
90	>	sys.exit()
91	>
92	>	# Read in Environment, XML and get optional Parameters
93	>
94	>	nJob       = int(os.environ.get('NJob',      '0'))
95	>	preserveSeeds  = os.environ.get('PreserveSeeds','')
96	>	incrementSeeds = os.environ.get('IncrementSeeds','')
97	>
98	>	# Defaults
99	>
100	>	maxEvents  = 0
101	>	skipEvents = 0
102	>	firstEvent = -1
103	>	compHEPFirstEvent = 0
104	>	firstRun   = 0
105	>	# FUTURE: Remove firstRun
106	>	firstLumi  = 0
107	>
108	>	dom = xml.dom.minidom.parse(os.environ['RUNTIME_AREA']+'/arguments.xml')
109	>
110	>	for elem in dom.getElementsByTagName("Job"):
111	>	if nJob == int(elem.getAttribute("JobID")):
112	>	if elem.getAttribute("MaxEvents"):
113	>	maxEvents = int(elem.getAttribute("MaxEvents"))
114	>	if elem.getAttribute("SkipEvents"):
115	>	skipEvents = int(elem.getAttribute("SkipEvents"))
116	>	if elem.getAttribute("FirstEvent"):
117	>	firstEvent = int(elem.getAttribute("FirstEvent"))
118	>	if elem.getAttribute("FirstRun"):
119	>	firstRun = int(elem.getAttribute("FirstRun"))
120	>	if elem.getAttribute("FirstLumi"):
121	>	firstLumi = int(elem.getAttribute("FirstLumi"))
122	>
123	>	generator      = str(elem.getAttribute('Generator'))
124	>	inputFiles     = str(elem.getAttribute('InputFiles'))
125	>	parentFiles    = str(elem.getAttribute('ParentFiles'))
126	>	lumis          = str(elem.getAttribute('Lumis'))
127
128	<	# Read Input cfg or python cfg file
128	>	# Read Input python config file
129
98	–	if fileName.endswith('py'):
130		handle = open(fileName, 'r')
131		try:   # Nested form for Python < 2.5
132	<	try:
133	<	cfo = imp.load_source("pycfg", fileName, handle)
134	<	cmsProcess = cfo.process
135	<	except Exception, ex:
136	<	msg = "Your pycfg file is not valid python: %s" % str(ex)
137	<	raise "Error: ",msg
132	>	try:
133	>	print "Importing .py file"
134	>	cfo = imp.load_source("pycfg", fileName, handle)
135	>	cmsProcess = cfo.process
136	>	except Exception, ex:
137	>	msg = "Your pycfg file is not valid python: %s" % str(ex)
138	>	raise ConfigException(msg)
139		finally:
140		handle.close()
109	–	else:
110	–	try:
111	–	cfo = include(fileName)
112	–	cmsProcess = cfo
113	–	except Exception, ex:
114	–	msg =  "The cfg file is not valid, %s\n" % str(ex)
115	–	raise "Error: ",msg
116	–	cfg = CfgInterface(cmsProcess)
117	–
118	–	# Set parameters for job
119	–	inModule = cfg.inputSource
120	–	if maxEvents:
121	–	cfg.maxEvents.setMaxEventsInput(maxEvents)
122	–
123	–	if skipEvents:
124	–	inModule.setSkipEvents(skipEvents)
125	–
126	–	if inputFiles:
127	–	inputFiles = inputFiles.replace('\\','')
128	–	inputFiles = inputFiles.replace('"','')
129	–	inputFileNames = inputFiles.split(',')
130	–	inModule.setFileNames(*inputFileNames)
131	–
132	–	# Pythia parameters
133	–	if (firstRun):
134	–	inModule.setFirstRun(firstRun)
135	–
136	–	incrementSeedList = []
137	–	preserveSeedList  = []
138	–
139	–	if incrementSeeds:
140	–	incrementSeedList = incrementSeeds.split(',')
141	–	if preserveSeeds:
142	–	preserveSeedList  = preserveSeeds.split(',')
143	–
144	–	# FUTURE: This function tests the CMSSW version. Can be simplified as we drop support for old versions
145	–	if CMSSW_major < 3: # True for now, should be < 2 when really ready
146	–	# Treatment for seeds, CMSSW < 2_0_x
147	–	if cfg.data.services.has_key('RandomNumberGeneratorService'):
148	–	ranGenerator = cfg.data.services['RandomNumberGeneratorService']
149	–	ranModules   = ranGenerator.moduleSeeds
141
142	<	_MAXINT = 900000000
142	>	cfg = CfgInterface(cmsProcess)
143
144	<	sourceSeed = int(ranGenerator.sourceSeed.value())
145	<	if 'sourceSeed' in preserveSeedList:
146	<	pass
147	<	elif 'sourceSeed' in incrementSeedList:
148	<	ranGenerator.sourceSeed = CfgTypes.untracked(CfgTypes.uint32(sourceSeed+nJob))
149	<	else:
150	<	ranGenerator.sourceSeed = CfgTypes.untracked(CfgTypes.uint32(_inst.randint(1,_MAXINT)))
151	<
152	<	for seed in incrementSeedList:
153	<	curSeed = getattr(ranGenerator.moduleSeeds,seed,None)
154	<	if curSeed:
155	<	curValue = int(curSeed.value())
156	<	setattr(ranGenerator.moduleSeeds,seed,CfgTypes.untracked(CfgTypes.uint32(curValue+nJob)))
157	<	preserveSeedList.append(seed)
158	<
159	<	for seed in ranGenerator.moduleSeeds.parameterNames_():
160	<	if seed not in preserveSeedList:
161	<	curSeed = getattr(ranGenerator.moduleSeeds,seed,None)
162	<	if curSeed:
163	<	curValue = int(curSeed.value())
164	<	setattr(ranGenerator.moduleSeeds,seed,CfgTypes.untracked(CfgTypes.uint32(_inst.randint(1,_MAXINT))))
165	<	else:
166	<	# Treatment for  seeds, CMSSW => 2_0_x
167	<	#from RandomService import RandomSeedService
144	>	# Set parameters for job
145	>	print "Setting parameters"
146	>	inModule = cfg.inputSource
147	>	if maxEvents:
148	>	cfg.maxEvents.setMaxEventsInput(maxEvents)
149	>
150	>	if skipEvents:
151	>	inModule.setSkipEvents(skipEvents)
152	>
153	>	# Set "skip events" for various generators
154	>	if generator == 'comphep':
155	>	cmsProcess.source.CompHEPFirstEvent = CfgTypes.int32(firstEvent)
156	>	elif generator == 'lhe':
157	>	cmsProcess.source.skipEvents = CfgTypes.untracked(CfgTypes.uint32(firstEvent))
158	>	cmsProcess.source.firstEvent = CfgTypes.untracked(CfgTypes.uint32(firstEvent+1))
159	>	elif firstEvent != -1: # (Old? Madgraph)
160	>	cmsProcess.source.firstEvent = CfgTypes.untracked(CfgTypes.uint32(firstEvent))
161	>
162	>	if inputFiles:
163	>	inputFileNames = inputFiles.split(',')
164	>	inModule.setFileNames(*inputFileNames)
165	>
166	>	# handle parent files if needed
167	>	if parentFiles:
168	>	parentFileNames = parentFiles.split(',')
169	>	inModule.setSecondaryFileNames(*parentFileNames)
170	>
171	>	if lumis:
172	>	if CMSSW_major < 3: # FUTURE: Can remove this check
173	>	print "Cannot skip lumis for CMSSW 2_x"
174	>	else:
175	>	lumiRanges = lumis.split(',')
176	>	inModule.setLumisToProcess(*lumiRanges)
177	>
178	>	# Pythia parameters
179	>	if (firstRun):
180	>	inModule.setFirstRun(firstRun)
181	>	if (firstLumi):
182	>	inModule.setFirstLumi(firstLumi)
183
184	+	# Check if there are random #'s to deal with
185	+	if cfg.data.services.has_key('RandomNumberGeneratorService'):
186	+	print "RandomNumberGeneratorService found, will attempt to change seeds"
187	+	from IOMC.RandomEngine.RandomServiceHelper import RandomNumberServiceHelper
188	+	ranGenerator = cfg.data.services['RandomNumberGeneratorService']
189	+	randSvc = RandomNumberServiceHelper(ranGenerator)
190	+
191	+	incrementSeedList = []
192	+	preserveSeedList  = []
193	+
194	+	if incrementSeeds:
195	+	incrementSeedList = incrementSeeds.split(',')
196	+	if preserveSeeds:
197	+	preserveSeedList  = preserveSeeds.split(',')
198	+
199	+	# Increment requested seed sets
200	+	for seedName in incrementSeedList:
201	+	curSeeds = randSvc.getNamedSeed(seedName)
202	+	newSeeds = [x+nJob for x in curSeeds]
203	+	randSvc.setNamedSeed(seedName, *newSeeds)
204	+	preserveSeedList.append(seedName)
205
206	<	# This code not currently working because randSvc is not part of the actual configuration file
206	>	# Randomize remaining seeds
207	>	randSvc.populate(*preserveSeedList)
208
209	<	# Translate old format to new format first
210	<	randSvc = RandomSeedService()
183	<	try:
184	<	ranGenerator = cfg.data.services['RandomNumberGeneratorService']
185	<	ranModules   = ranGenerator.moduleSeeds
186	<	for seed in ranGenerator.moduleSeeds.parameters().keys():
187	<	curSeed = getattr(ranGenerator.moduleSeeds,seed,None)
188	<	curValue = int(curSeed.value())
189	<	setattr(randSvc,seed,CfgTypes.PSet())
190	<	curPSet = getattr(randSvc,seed,None)
191	<	curPSet.initialSeed = CfgTypes.untracked(CfgTypes.uint32(curValue))
192	<	del ranGenerator.moduleSeeds # Get rid of seeds in old format
193	<	# Doesn't work, filter is false      randSvc.populate()
194	<
195	<	except:
196	<	print "Problems converting old seeds to new format"
197	<
198	<	# Write out new config file in one format or the other
199	<	outFile = open(outFileName,"w")
200	<	if (outFileName.endswith('py') or outFileName.endswith('pycfg') ):
209	>	# Write out new config file
210	>	outFile = open(outFileName,"w")
211		outFile.write("import FWCore.ParameterSet.Config as cms\n")
212	<	outFile.write(cmsProcess.dumpPython())
212	>	outFile.write("import pickle\n")
213	>	outFile.write("pickledCfg=\"\"\"%s\"\"\"\n" % pickle.dumps(cmsProcess))
214	>	outFile.write("process = pickle.loads(pickledCfg)\n")
215	>	outFile.close()
216		if (debug):
217	<	print "writeCfg output:"
218	<	print "import FWCore.ParameterSet.Config as cms"
219	<	print cmsProcess.dumpPython()
207	<	else:
208	<	outFile.write(str(cfg))
209	<	if (debug):
210	<	print "writeCfg output:"
211	<	print str(cfg)
212	<	outFile.close()
217	>	print "writeCfg output (May not be exact):"
218	>	print "import FWCore.ParameterSet.Config as cms"
219	>	print cmsProcess.dumpPython()
220
221
222		if __name__ == '__main__' :
223		exit_status = main(sys.argv[1:])
224		sys.exit(exit_status)
218	–

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines