ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/writeCfg.py

Comparing COMP/CRAB/python/writeCfg.py (file contents):
Revision 1.1 by ewv, Fri Dec 14 17:53:36 2007 UTC vs.
Revision 1.30 by ewv, Wed Oct 6 13:57:46 2010 UTC

#	Line 1 | Line 1
1		#!/usr/bin/env python
2
3	<	import sys, getopt, string
3	>	"""
4	>	Re-write config file and optionally convert to python
5	>	"""
6	>
7	>	__revision__ = "$Id$"
8	>	__version__ = "$Revision$"
9	>
10	>	import getopt
11		import imp
12		import os
13	+	import pickle
14	+	import sys
15	+	import xml.dom.minidom
16	+
17	+	from random import SystemRandom
18
19		from ProdCommon.CMSConfigTools.ConfigAPI.CfgInterface import CfgInterface
20	<	from FWCore.ParameterSet.DictTypes import SortedKeysDict
21	<	from FWCore.ParameterSet.Modules   import Service
22	<	from FWCore.ParameterSet.Types     import *
11	<	from FWCore.ParameterSet.Config import include
20	>	import FWCore.ParameterSet.Types as CfgTypes
21	>
22	>	MyRandom  = SystemRandom()
23
24	<	import FWCore.ParameterSet.Types   as CfgTypes
25	<	import FWCore.ParameterSet.Modules as CfgModules
24	>	class ConfigException(Exception):
25	>	"""
26	>	Exceptions raised by writeCfg
27	>	"""
28	>
29	>	def __init__(self, msg):
30	>	Exception.__init__(self, msg)
31	>	self._msg = msg
32	>	return
33
34	+	def __str__(self):
35	+	return self._msg
36
37		def main(argv) :
38	<	"""
39	<	writeCfg
38	>	"""
39	>	writeCfg
40
41	<	- Read in existing, user supplied cfg or pycfg file
42	<	- Modify job specific parameters based on environment variables
43	<	- Write out modified cfg or pycfg file
44	<
45	<	required parameters: none
46	<
47	<	optional parameters:
48	<	--help             :       help
49	<	--debug            :       debug statements
50	<
51	<	"""
52	<
53	<	# defaults
54	<	maxEvents = 0
55	<	skipEvents = 0
56	<	inputFileNames = None
57	<	debug = False
38	<
39	<	try:
40	<	opts, args = getopt.getopt(argv, "", ["debug", "help","inputFiles=","maxEvents=","skipEvents="])
41	<	except getopt.GetoptError:
42	<	print main.__doc__
43	<	sys.exit(2)
44	<
45	<	# Parse command line parameters
46	<	for opt, arg in opts :
47	<	if opt  == "--help" :
48	<	print main.__doc__
49	<	sys.exit()
50	<	elif opt == "--debug" :
51	<	debug = True
52	<	elif opt == "--maxEvents":
53	<	maxEvents = int(arg)
54	<	elif opt == "--skipEvents":
55	<	skipEvents = int(arg)
56	<	elif opt == "--inputFiles":
57	<	inputFiles = arg
58	<	inputFiles = inputFiles.replace('\\','')
59	<	inputFiles = inputFiles.replace('"','')
60	<	inputFileNames = inputFiles.split(',')
41	>	- Read in existing, user supplied pycfg or pickled pycfg file
42	>	- Modify job specific parameters based on environment variables and arguments.xml
43	>	- Write out pickled pycfg file
44	>
45	>	required parameters: none
46	>
47	>	optional parameters:
48	>	--help             :       help
49	>	--debug            :       debug statements
50	>
51	>	"""
52	>
53	>	# defaults
54	>	inputFileNames  = None
55	>	parentFileNames = None
56	>	debug           = False
57	>	_MAXINT         = 900000000
58
59	<	# Parse remaining parameters
59	>	try:
60	>	opts, args = getopt.getopt(argv, "", ["debug", "help"])
61	>	except getopt.GetoptError:
62	>	print main.__doc__
63	>	sys.exit(2)
64	>
65	>	try:
66	>	CMSSW  = os.environ['CMSSW_VERSION']
67	>	parts = CMSSW.split('_')
68	>	CMSSW_major = int(parts[1])
69	>	CMSSW_minor = int(parts[2])
70	>	CMSSW_patch = int(parts[3])
71	>	except (KeyError, ValueError):
72	>	msg = "Your environment doesn't specify the CMSSW version or specifies it incorrectly"
73	>	raise ConfigException(msg)
74	>
75	>	# Parse command line options
76	>	for opt, arg in opts :
77	>	if opt  == "--help" :
78	>	print main.__doc__
79	>	sys.exit()
80	>	elif opt == "--debug" :
81	>	debug = True
82
83	<	fileName    = args[0];
84	<	outFileName = args[1];
83	>	# Parse remaining parameters
84	>	try:
85	>	fileName    = args[0]
86	>	outFileName = args[1]
87	>	except IndexError:
88	>	print main.__doc__
89	>	sys.exit()
90	>
91	>	# Read in Environment, XML and get optional Parameters
92	>
93	>	nJob       = int(os.environ.get('NJob',      '0'))
94	>	preserveSeeds  = os.environ.get('PreserveSeeds','')
95	>	incrementSeeds = os.environ.get('IncrementSeeds','')
96	>
97	>	# Defaults
98	>
99	>	maxEvents  = 0
100	>	skipEvents = 0
101	>	firstEvent = -1
102	>	compHEPFirstEvent = 0
103	>	firstRun   = 0
104	>	# FUTURE: Remove firstRun
105	>	firstLumi  = 0
106	>
107	>	dom = xml.dom.minidom.parse(os.environ['RUNTIME_AREA']+'/arguments.xml')
108	>
109	>	for elem in dom.getElementsByTagName("Job"):
110	>	if nJob == int(elem.getAttribute("JobID")):
111	>	if elem.getAttribute("MaxEvents"):
112	>	maxEvents = int(elem.getAttribute("MaxEvents"))
113	>	if elem.getAttribute("SkipEvents"):
114	>	skipEvents = int(elem.getAttribute("SkipEvents"))
115	>	if elem.getAttribute("FirstEvent"):
116	>	firstEvent = int(elem.getAttribute("FirstEvent"))
117	>	if elem.getAttribute("FirstRun"):
118	>	firstRun = int(elem.getAttribute("FirstRun"))
119	>	if elem.getAttribute("FirstLumi"):
120	>	firstLumi = int(elem.getAttribute("FirstLumi"))
121	>
122	>	generator      = str(elem.getAttribute('Generator'))
123	>	inputFiles     = str(elem.getAttribute('InputFiles'))
124	>	parentFiles    = str(elem.getAttribute('ParentFiles'))
125	>	lumis          = str(elem.getAttribute('Lumis'))
126
127	<	# Input cfg or python cfg file
127	>	# Read Input python config file
128
69	–	if (fileName.endswith('py') or fileName.endswith('pycfg') ):
129		handle = open(fileName, 'r')
130		try:   # Nested form for Python < 2.5
131	<	try:
132	<	cfo = imp.load_source("pycfg", fileName, handle)
133	<	except Exception, ex:
134	<	msg = "Your pycfg file is not valid python: %s" % str(ex)
135	<	raise "Error: ",msg
131	>	try:
132	>	print "Importing .py file"
133	>	cfo = imp.load_source("pycfg", fileName, handle)
134	>	cmsProcess = cfo.process
135	>	except Exception, ex:
136	>	msg = "Your pycfg file is not valid python: %s" % str(ex)
137	>	raise ConfigException(msg)
138		finally:
139		handle.close()
79	–	cfg = CfgInterface(cfo.process)
80	–	else:
81	–	try:
82	–	cfo = include(fileName)
83	–	cfg = CfgInterface(cfo)
84	–	except Exception, ex:
85	–	msg =  "The cfg file is not valid, %s\n" % str(ex)
86	–	raise "Error: ",msg
87	–
88	–	# Set parameters for job
89	–
90	–	inModule = cfg.inputSource
91	–
92	–	cfg.maxEvents.setMaxEventsInput(maxEvents)
140
141	<	inModule.setSkipEvents(skipEvents)
95	<	if (inputFileNames):
96	<	inModule.setFileNames(*inputFileNames)
141	>	cfg = CfgInterface(cmsProcess)
142
143	<	# Write out new config file
144	<
145	<	outFile = open(outFileName,"w")
146	<	outFile.write("import FWCore.ParameterSet.Config as cms\n")
147	<	outFile.write(cfo.dumpPython())
148	<	outFile.close()
149	<
150	<	if (debug):
151	<	print "writeCfg output:"
152	<	print "import FWCore.ParameterSet.Config as cms"
153	<	print cfo.dumpPython()
143	>	# Set parameters for job
144	>	print "Setting parameters"
145	>	inModule = cfg.inputSource
146	>	if maxEvents:
147	>	cfg.maxEvents.setMaxEventsInput(maxEvents)
148	>
149	>	if skipEvents and inModule.sourceType not in ['EmptySource']:
150	>	inModule.setSkipEvents(skipEvents)
151	>
152	>	# Set "skip events" for various generators
153	>	if generator == 'comphep':
154	>	cmsProcess.source.CompHEPFirstEvent = CfgTypes.int32(firstEvent)
155	>	elif generator == 'lhe':
156	>	cmsProcess.source.skipEvents = CfgTypes.untracked(CfgTypes.uint32(firstEvent))
157	>	cmsProcess.source.firstEvent = CfgTypes.untracked(CfgTypes.uint32(firstEvent+1))
158	>	elif firstEvent != -1: # (Old? Madgraph)
159	>	cmsProcess.source.firstEvent = CfgTypes.untracked(CfgTypes.uint32(firstEvent))
160	>
161	>	if inputFiles:
162	>	inputFileNames = inputFiles.split(',')
163	>	inModule.setFileNames(*inputFileNames)
164	>
165	>	# handle parent files if needed
166	>	if parentFiles:
167	>	parentFileNames = parentFiles.split(',')
168	>	inModule.setSecondaryFileNames(*parentFileNames)
169	>
170	>	if lumis:
171	>	if CMSSW_major < 3: # FUTURE: Can remove this check
172	>	print "Cannot skip lumis for CMSSW 2_x"
173	>	else:
174	>	lumiRanges = lumis.split(',')
175	>	inModule.setLumisToProcess(*lumiRanges)
176	>
177	>	# Pythia parameters
178	>	if (firstRun):
179	>	inModule.setFirstRun(firstRun)
180	>	if (firstLumi):
181	>	inModule.setFirstLumi(firstLumi)
182	>
183	>	# Check if there are random #'s to deal with
184	>	if cfg.data.services.has_key('RandomNumberGeneratorService'):
185	>	print "RandomNumberGeneratorService found, will attempt to change seeds"
186	>	from IOMC.RandomEngine.RandomServiceHelper import RandomNumberServiceHelper
187	>	ranGenerator = cfg.data.services['RandomNumberGeneratorService']
188	>
189	>	ranModules   = getattr(ranGenerator, "moduleSeeds", None)
190	>	oldSource    = getattr(ranGenerator, "sourceSeed",  None)
191	>	if ranModules != None or oldSource != None:
192	>	msg = "Your random number seeds are set in an old,\n"
193	>	msg += "deprecated style. Please change to new style:\n"
194	>	msg += "https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideEDMRandomNumberGeneratorService"
195	>	raise ConfigException(msg)
196	>
197	>	randSvc = RandomNumberServiceHelper(ranGenerator)
198	>
199	>	incrementSeedList = []
200	>	preserveSeedList  = []
201	>
202	>	if incrementSeeds:
203	>	incrementSeedList = incrementSeeds.split(',')
204	>	if preserveSeeds:
205	>	preserveSeedList  = preserveSeeds.split(',')
206	>
207	>	# Increment requested seed sets
208	>	for seedName in incrementSeedList:
209	>	curSeeds = randSvc.getNamedSeed(seedName)
210	>	newSeeds = [x+nJob for x in curSeeds]
211	>	randSvc.setNamedSeed(seedName, *newSeeds)
212	>	preserveSeedList.append(seedName)
213	>
214	>	# Randomize remaining seeds
215	>	randSvc.populate(*preserveSeedList)
216	>
217	>	# Write out new config file
218	>	pklFileName = outFileName + '.pkl'
219	>	outFile = open(outFileName,"w")
220	>	outFile.write("import FWCore.ParameterSet.Config as cms\n")
221	>	outFile.write("import pickle\n")
222	>	outFile.write("process = pickle.load(open('%s', 'rb'))\n" % pklFileName)
223	>	outFile.close()
224	>
225	>	pklFile = open(pklFileName,"wb")
226	>	myPickle = pickle.Pickler(pklFile)
227	>	myPickle.dump(cmsProcess)
228	>	pklFile.close()
229	>
230	>	if (debug):
231	>	print "writeCfg output (May not be exact):"
232	>	print "import FWCore.ParameterSet.Config as cms"
233	>	print cmsProcess.dumpPython()
234
235
236		if __name__ == '__main__' :
237		exit_status = main(sys.argv[1:])
238		sys.exit(exit_status)
114	–

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines