ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/writeCfg.py

Comparing COMP/CRAB/python/writeCfg.py (file contents):
Revision 1.9 by ewv, Wed May 7 16:30:35 2008 UTC vs.
Revision 1.33 by spiga, Wed Mar 23 16:48:53 2011 UTC

#	Line 1 | Line 1
1		#!/usr/bin/env python
2
3	<	import sys, getopt, string
3	>	"""
4	>	Re-write config file and optionally convert to python
5	>	"""
6	>
7	>	__revision__ = "$Id$"
8	>	__version__ = "$Revision$"
9	>
10	>	import getopt
11		import imp
12		import os
13	<	import random
13	>	import pickle
14	>	import sys
15	>	import xml.dom.minidom
16	>
17		from random import SystemRandom
8	–	_inst  = SystemRandom()
18
19		from ProdCommon.CMSConfigTools.ConfigAPI.CfgInterface import CfgInterface
20	<	from FWCore.ParameterSet.DictTypes import SortedKeysDict
12	<	from FWCore.ParameterSet.Modules   import Service
13	<	from FWCore.ParameterSet.Types     import *
14	<	from FWCore.ParameterSet.Config    import include
20	>	import FWCore.ParameterSet.Types as CfgTypes
21
22	<	import FWCore.ParameterSet.Types   as CfgTypes
17	<	import FWCore.ParameterSet.Modules as CfgModules
22	>	MyRandom  = SystemRandom()
23
24		class ConfigException(Exception):
25	+	"""
26	+	Exceptions raised by writeCfg
27	+	"""
28	+
29		def __init__(self, msg):
30		Exception.__init__(self, msg)
31		self._msg = msg
#	Line 26 | Line 35 | class ConfigException(Exception):
35		return self._msg
36
37		def main(argv) :
38	<	"""
39	<	writeCfg
38	>	"""
39	>	writeCfg
40	>
41	>	- Read in existing, user supplied pycfg or pickled pycfg file
42	>	- Modify job specific parameters based on environment variables and arguments.xml
43	>	- Write out pickled pycfg file
44	>
45	>	required parameters: none
46	>
47	>	optional parameters:
48	>	--help             :       help
49	>	--debug            :       debug statements
50	>
51	>	"""
52	>
53	>	# defaults
54	>	inputFileNames  = None
55	>	parentFileNames = None
56	>	debug           = False
57	>	_MAXINT         = 900000000
58	>
59	>	try:
60	>	opts, args = getopt.getopt(argv, "", ["debug", "help"])
61	>	except getopt.GetoptError:
62	>	print main.__doc__
63	>	sys.exit(2)
64	>
65	>	try:
66	>	CMSSW  = os.environ['CMSSW_VERSION']
67	>	parts = CMSSW.split('_')
68	>	CMSSW_major = int(parts[1])
69	>	CMSSW_minor = int(parts[2])
70	>	CMSSW_patch = int(parts[3])
71	>	except (KeyError, ValueError):
72	>	msg = "Your environment doesn't specify the CMSSW version or specifies it incorrectly"
73	>	raise ConfigException(msg)
74	>
75	>	# Parse command line options
76	>	for opt, arg in opts :
77	>	if opt  == "--help" :
78	>	print main.__doc__
79	>	sys.exit()
80	>	elif opt == "--debug" :
81	>	debug = True
82	>
83	>	# Parse remaining parameters
84	>	try:
85	>	fileName    = args[0]
86	>	outFileName = args[1]
87	>	except IndexError:
88	>	print main.__doc__
89	>	sys.exit()
90	>
91	>	# Read in Environment, XML and get optional Parameters
92	>
93	>	nJob       = int(os.environ.get('NJob',      '0'))
94	>	preserveSeeds  = os.environ.get('PreserveSeeds','')
95	>	incrementSeeds = os.environ.get('IncrementSeeds','')
96	>
97	>	# Defaults
98	>
99	>	maxEvents  = 0
100	>	skipEvents = 0
101	>	firstEvent = -1
102	>	compHEPFirstEvent = 0
103	>	firstRun   = 0
104	>	# FUTURE: Remove firstRun
105	>	firstLumi  = 0
106	>
107	>	dom = xml.dom.minidom.parse(os.environ['RUNTIME_AREA']+'/arguments.xml')
108	>
109	>	for elem in dom.getElementsByTagName("Job"):
110	>	if nJob == int(elem.getAttribute("JobID")):
111	>	if elem.getAttribute("MaxEvents"):
112	>	maxEvents = int(elem.getAttribute("MaxEvents"))
113	>	if elem.getAttribute("SkipEvents"):
114	>	skipEvents = int(elem.getAttribute("SkipEvents"))
115	>	if elem.getAttribute("FirstEvent"):
116	>	firstEvent = int(elem.getAttribute("FirstEvent"))
117	>	if elem.getAttribute("FirstRun"):
118	>	firstRun = int(elem.getAttribute("FirstRun"))
119	>	if elem.getAttribute("FirstLumi"):
120	>	firstLumi = int(elem.getAttribute("FirstLumi"))
121	>
122	>	generator      = str(elem.getAttribute('Generator'))
123	>	inputBlocks    = str(elem.getAttribute('InputBlocks'))
124	>	inputFiles     = str(elem.getAttribute('InputFiles'))
125	>	parentFiles    = str(elem.getAttribute('ParentFiles'))
126	>	lumis          = str(elem.getAttribute('Lumis'))
127
128	<	- Read in existing, user supplied cfg or pycfg file
33	<	- Modify job specific parameters based on environment variables
34	<	- Write out modified cfg or pycfg file
35	<
36	<	required parameters: none
37	<
38	<	optional parameters:
39	<	--help             :       help
40	<	--debug            :       debug statements
41	<
42	<	"""
43	<
44	<	# defaults
45	<	inputFileNames = None
46	<	firstRun       = 0
47	<	sourceSeed     = 0
48	<	vtxSeed        = 0
49	<	g4Seed         = 0
50	<	mixSeed        = 0
51	<	debug          = False
52	<	_MAXINT        = 900000000
53	<	maxSeeds       = 4         # Kludge, maximum # of seeds that any engine takes
54	<
55	<	try:
56	<	opts, args = getopt.getopt(argv, "", ["debug", "help"])
57	<	except getopt.GetoptError:
58	<	print main.__doc__
59	<	sys.exit(2)
60	<
61	<	try:
62	<	CMSSW  = os.environ['CMSSW_VERSION']
63	<	parts = CMSSW.split('_')
64	<	CMSSW_major = int(parts[1])
65	<	CMSSW_minor = int(parts[2])
66	<	CMSSW_patch = int(parts[3])
67	<	except KeyError, ValueError:
68	<	msg = "Your environment doesn't specify the CMSSW version or specifies it incorrectly"
69	<	raise ConfigException(msg)
70	<
71	<	# Parse command line options
72	<	for opt, arg in opts :
73	<	if opt  == "--help" :
74	<	print main.__doc__
75	<	sys.exit()
76	<	elif opt == "--debug" :
77	<	debug = True
78	<
79	<	# Parse remaining parameters
80	<
81	<	try:
82	<	fileName    = args[0];
83	<	outFileName = args[1];
84	<	except IndexError:
85	<	print main.__doc__
86	<	sys.exit()
87	<
88	<	# Optional Parameters
89	<
90	<	maxEvents  = int(os.environ.get('MaxEvents', '0'))
91	<	skipEvents = int(os.environ.get('SkipEvents','0'))
92	<	firstRun   = int(os.environ.get('FirstRun',  '0'))
93	<	nJob       = int(os.environ.get('NJob',      '0'))
94	<
95	<	inputFiles     = os.environ.get('InputFiles','')
96	<	preserveSeeds  = os.environ.get('PreserveSeeds','')
97	<	incrementSeeds = os.environ.get('IncrementSeeds','')
128	>	report(inputBlocks,inputFiles,parentFiles,lumis)
129
130	<	# Read Input cfg or python cfg file, FUTURE: Get rid cfg mode
130	>	# Read Input python config file
131
101	–	if fileName.endswith('py'):
132		handle = open(fileName, 'r')
133		try:   # Nested form for Python < 2.5
134	<	try:
135	<	cfo = imp.load_source("pycfg", fileName, handle)
136	<	cmsProcess = cfo.process
137	<	except Exception, ex:
138	<	msg = "Your pycfg file is not valid python: %s" % str(ex)
139	<	raise "Error: ",msg
134	>	try:
135	>	print "Importing .py file"
136	>	cfo = imp.load_source("pycfg", fileName, handle)
137	>	cmsProcess = cfo.process
138	>	except Exception, ex:
139	>	msg = "Your pycfg file is not valid python: %s" % str(ex)
140	>	raise ConfigException(msg)
141		finally:
142		handle.close()
143	<	else:
144	<	try:
145	<	cfo = include(fileName)
146	<	cmsProcess = cfo
147	<	except Exception, ex:
148	<	msg =  "The cfg file is not valid, %s\n" % str(ex)
149	<	raise "Error: ",msg
150	<	cfg = CfgInterface(cmsProcess)
151	<
152	<	# Set parameters for job
153	<	inModule = cfg.inputSource
154	<	if maxEvents:
155	<	cfg.maxEvents.setMaxEventsInput(maxEvents)
156	<
157	<	if skipEvents:
158	<	inModule.setSkipEvents(skipEvents)
159	<
160	<	if inputFiles:
161	<	inputFiles = inputFiles.replace('\\','')
162	<	inputFiles = inputFiles.replace('"','')
163	<	inputFileNames = inputFiles.split(',')
164	<	inModule.setFileNames(*inputFileNames)
165	<
166	<	# Pythia parameters
167	<	if (firstRun):
168	<	inModule.setFirstRun(firstRun)
169	<
170	<	incrementSeedList = []
171	<	preserveSeedList  = []
172	<
173	<	if incrementSeeds:
174	<	incrementSeedList = incrementSeeds.split(',')
175	<	if preserveSeeds:
176	<	preserveSeedList  = preserveSeeds.split(',')
177	<
178	<	# FUTURE: This function tests the CMSSW version and presence of old-style seed specification. Reduce when we drop support for old versions
179	<	if cfg.data.services.has_key('RandomNumberGeneratorService'): # There are random #'s to deal with
180	<	print "RandomNumberGeneratorService found, will attempt to change seeds"
181	<	ranGenerator = cfg.data.services['RandomNumberGeneratorService']
182	<	ranModules   = getattr(ranGenerator,"moduleSeeds",None)
183	<	if ranModules != None:              # Old format present, no matter the CMSSW version
184	<	print "Old-style random number seeds found, will be changed."
185	<	sourceSeed = int(ranGenerator.sourceSeed.value())
186	<	if ('sourceSeed' in preserveSeedList) or ('theSource' in preserveSeedList):
187	<	pass
188	<	elif ('sourceSeed' in incrementSeedList) or ('theSource' in incrementSeedList):
189	<	ranGenerator.sourceSeed = CfgTypes.untracked(CfgTypes.uint32(sourceSeed+nJob))
190	<	else:
191	<	ranGenerator.sourceSeed = CfgTypes.untracked(CfgTypes.uint32(_inst.randint(1,_MAXINT)))
192	<
193	<	for seed in incrementSeedList:
194	<	curSeed = getattr(ranGenerator.moduleSeeds,seed,None)
195	<	if curSeed:
196	<	curValue = int(curSeed.value())
197	<	setattr(ranGenerator.moduleSeeds,seed,CfgTypes.untracked(CfgTypes.uint32(curValue+nJob)))
198	<	preserveSeedList.append(seed)
199	<
200	<	for seed in ranGenerator.moduleSeeds.parameterNames_():
201	<	if seed not in preserveSeedList:
202	<	curSeed = getattr(ranGenerator.moduleSeeds,seed,None)
203	<	if curSeed:
204	<	curValue = int(curSeed.value())
205	<	setattr(ranGenerator.moduleSeeds,seed,CfgTypes.untracked(CfgTypes.uint32(_inst.randint(1,_MAXINT))))
206	<	elif CMSSW_major > 2 or (CMSSW_major == 2 and CMSSW_minor >= 1): # Treatment for  seeds, CMSSW 2_1_x and later
207	<	print "New-style random number seeds found, will be changed."
208	<	from IOMC.RandomEngine.RandomServiceHelper import RandomNumberServiceHelper
209	<	randSvc = RandomNumberServiceHelper(ranGenerator)
210	<
211	<	# Increment requested seed sets
212	<	for seedName in incrementSeedList:
213	<	curSeeds = randSvc.getNamedSeed(seedName)
214	<	newSeeds = [x+nJob for x in curSeeds]
215	<	randSvc.setNamedSeed(seedName,*newSeeds)
216	<	preserveSeedList.append(seedName)
217	<
218	<	# Randomize remaining seeds
219	<	randSvc.populate(*preserveSeedList)
220	<	else:
221	<	print "Neither old nor new seed format found!"
222	<
192	<	# End version specific code
193	<
194	<	# Write out new config file in one format or the other, FUTURE: Get rid of cfg mode
195	<	outFile = open(outFileName,"w")
196	<	if outFileName.endswith('py'):
143	>
144	>	cfg = CfgInterface(cmsProcess)
145	>
146	>	# Set parameters for job
147	>	print "Setting parameters"
148	>	inModule = cfg.inputSource
149	>	if maxEvents:
150	>	cfg.maxEvents.setMaxEventsInput(maxEvents)
151	>
152	>	if skipEvents and inModule.sourceType not in ['EmptySource']:
153	>	inModule.setSkipEvents(skipEvents)
154	>
155	>	# Set "skip events" for various generators
156	>	if generator == 'comphep':
157	>	cmsProcess.source.CompHEPFirstEvent = CfgTypes.int32(firstEvent)
158	>	elif generator == 'lhe':
159	>	cmsProcess.source.skipEvents = CfgTypes.untracked(CfgTypes.uint32(firstEvent))
160	>	cmsProcess.source.firstEvent = CfgTypes.untracked(CfgTypes.uint32(firstEvent+1))
161	>	elif firstEvent != -1: # (Old? Madgraph)
162	>	cmsProcess.source.firstEvent = CfgTypes.untracked(CfgTypes.uint32(firstEvent))
163	>
164	>	if inputFiles:
165	>	inputFileNames = inputFiles.split(',')
166	>	inModule.setFileNames(*inputFileNames)
167	>
168	>	# handle parent files if needed
169	>	if parentFiles:
170	>	parentFileNames = parentFiles.split(',')
171	>	inModule.setSecondaryFileNames(*parentFileNames)
172	>
173	>	if lumis:
174	>	if CMSSW_major < 3: # FUTURE: Can remove this check
175	>	print "Cannot skip lumis for CMSSW 2_x"
176	>	else:
177	>	lumiRanges = lumis.split(',')
178	>	inModule.setLumisToProcess(*lumiRanges)
179	>
180	>	# Pythia parameters
181	>	if (firstRun):
182	>	inModule.setFirstRun(firstRun)
183	>	if (firstLumi):
184	>	inModule.setFirstLumi(firstLumi)
185	>
186	>	# Check if there are random #'s to deal with
187	>	if cfg.data.services.has_key('RandomNumberGeneratorService'):
188	>	print "RandomNumberGeneratorService found, will attempt to change seeds"
189	>	from IOMC.RandomEngine.RandomServiceHelper import RandomNumberServiceHelper
190	>	ranGenerator = cfg.data.services['RandomNumberGeneratorService']
191	>
192	>	ranModules   = getattr(ranGenerator, "moduleSeeds", None)
193	>	oldSource    = getattr(ranGenerator, "sourceSeed",  None)
194	>	if ranModules != None or oldSource != None:
195	>	msg = "Your random number seeds are set in an old,\n"
196	>	msg += "deprecated style. Please change to new style:\n"
197	>	msg += "https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideEDMRandomNumberGeneratorService"
198	>	raise ConfigException(msg)
199	>
200	>	randSvc = RandomNumberServiceHelper(ranGenerator)
201	>
202	>	incrementSeedList = []
203	>	preserveSeedList  = []
204	>
205	>	if incrementSeeds:
206	>	incrementSeedList = incrementSeeds.split(',')
207	>	if preserveSeeds:
208	>	preserveSeedList  = preserveSeeds.split(',')
209	>
210	>	# Increment requested seed sets
211	>	for seedName in incrementSeedList:
212	>	curSeeds = randSvc.getNamedSeed(seedName)
213	>	newSeeds = [x+nJob for x in curSeeds]
214	>	randSvc.setNamedSeed(seedName, *newSeeds)
215	>	preserveSeedList.append(seedName)
216	>
217	>	# Randomize remaining seeds
218	>	randSvc.populate(*preserveSeedList)
219	>
220	>	# Write out new config file
221	>	pklFileName = outFileName + '.pkl'
222	>	outFile = open(outFileName,"w")
223		outFile.write("import FWCore.ParameterSet.Config as cms\n")
224	<	outFile.write(cmsProcess.dumpPython())
225	<	if (debug):
226	<	print "writeCfg output:"
227	<	print "import FWCore.ParameterSet.Config as cms"
228	<	print cmsProcess.dumpPython()
229	<	else:
230	<	outFile.write(str(cfg))
224	>	outFile.write("import pickle\n")
225	>	outFile.write("process = pickle.load(open('%s', 'rb'))\n" % pklFileName)
226	>	outFile.close()
227	>
228	>	pklFile = open(pklFileName,"wb")
229	>	myPickle = pickle.Pickler(pklFile)
230	>	myPickle.dump(cmsProcess)
231	>	pklFile.close()
232	>
233		if (debug):
234	<	print "writeCfg output:"
235	<	print str(cfg)
236	<	outFile.close()
234	>	print "writeCfg output (May not be exact):"
235	>	print "import FWCore.ParameterSet.Config as cms"
236	>	print cmsProcess.dumpPython()
237	>
238	>
239	>	def report( inputBlocks='', inputFiles='', parentFiles='', lumis='' ):
240	>	"""
241	>	Writes the 4 parameters to a file, one parameter per line.
242	>	"""
243	>	outFile = open('%s/inputsReport.txt'%os.environ['RUNTIME_AREA'],"a")
244	>
245	>	#  InputFileList=inputFiles.split(',')
246	>	#  parentFilesList= parentFiles.split(',')
247	>	#  lumisList= lumis.split(',')
248	>
249	>	## replacing , with ; otherwise report.py will split it as a new parameter
250	>	txt = ''
251	>	txt += 'inputBlocks='+inputBlocks.replace(',',';')+'\n'
252	>	txt += 'inputFiles='+inputFiles.replace(',',';')+'\n'
253	>	txt += 'parentFiles='+parentFiles.replace(',',';')+'\n'
254	>	txt += 'lumisRange='+lumis.replace(',',';')+'\n'
255	>	outFile.write(str(txt))
256	>	outFile.close()
257	>	return
258
259
260		if __name__ == '__main__' :

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines