ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/UserCode/VHbb/python/submitThem.py

Comparing UserCode/VHbb/python/submitThem.py (file contents):
Revision 1.24 by peller, Wed Jan 16 16:22:47 2013 UTC vs.
Revision 1.37 by bortigno, Wed Mar 27 14:29:38 2013 UTC

#	Line 1 | Line 1
1		#! /usr/bin/env python
2	–	import os,shutil,sys,pickle,subprocess,ROOT
2		from optparse import OptionParser
3	<	import getpass
4	<	from myutils import BetterConfigParser, sample, parse_info
3	>	import sys
4	>	import time
5	>	import os
6	>	import shutil
7
8		parser = OptionParser()
9		parser.add_option("-T", "--tag", dest="tag", default="",
#	Line 13 | Line 14 | parser.add_option("-M", "--mass", dest="
14		help="Mass for DC or Plots, 110...135")
15		parser.add_option("-S","--samples",dest="samples",default="",
16		help="samples you want to run on")
17	<
17	>	parser.add_option("-F", "--folderTag", dest="ftag", default="",
18	>	help="Creats a new folder structure for outputs or uses an existing one with the given name")
19	>	parser.add_option("-N", "--number-of-events", dest="nevents_split", default=100000,
20	>	help="Number of events per file when splitting.")
21	>	parser.add_option("-P", "--philipp-love-progress-bars", dest="philipp_love_progress_bars", default=False,
22	>	help="If you share the love of Philipp...")
23
24		(opts, args) = parser.parse_args(sys.argv)
25	+
26	+	import os,shutil,pickle,subprocess,ROOT,re
27	+	ROOT.gROOT.SetBatch(True)
28	+	from myutils import BetterConfigParser, Sample, ParseInfo, sample_parser
29	+	import getpass
30	+
31		if opts.tag == "":
32		print "Please provide tag to run the analysis with, example '-T 8TeV' uses config8TeV and pathConfig8TeV to run the analysis."
33		sys.exit(123)
#	Line 24 | Line 36 | if opts.task == "":
36		print "Please provide a task.\n-J prep:\tpreparation of Trees\n-J sys:\t\twrite regression and systematics\n-J eval:\tcreate MVA output\n-J plot:\tproduce Plots\n-J dc:\t\twrite workspaces and datacards"
37		sys.exit(123)
38
39	+
40	+	en = opts.tag
41	+
42		#create the list with the samples to run over
43		samplesList=opts.samples.split(",")
44	+	timestamp = time.asctime().replace(' ','_').replace(':','-')
45	+
46	+	# the list of the config is taken from the path config
47	+	pathconfig = BetterConfigParser()
48	+	pathconfig.read('%sconfig/paths'%(en))
49	+	_configs = pathconfig.get('Configuration','List').split(" ")
50	+	configs = [ '%sconfig/'%(en) + c for c in _configs  ]
51	+
52	+	if not opts.ftag == '':
53	+	tagDir = pathconfig.get('Directories','tagDir')
54	+	DirStruct={'tagDir':tagDir,'ftagdir':'%s/%s/'%(tagDir,opts.ftag),'logpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Logs'),'plotpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Plots'),'limitpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Limits'),'confpath':'%s/%s/%s/'%(tagDir,opts.ftag,'config') }
55	+
56	+	for keys in ['tagDir','ftagdir','logpath','plotpath','limitpath','confpath']:
57	+	try:
58	+	os.stat(DirStruct[keys])
59	+	except:
60	+	os.mkdir(DirStruct[keys])
61	+
62	+	pathfile = open('%sconfig/paths'%(en))
63	+	buffer = pathfile.readlines()
64	+	pathfile.close()
65	+	os.rename('%sconfig/paths'%(en),'%sconfig/paths.bkp'%(en))
66	+	pathfile = open('%sconfig/paths'%(en),'w')
67	+	for line in buffer:
68	+	if line.startswith('plotpath'):
69	+	line = 'plotpath: %s\n'%DirStruct['plotpath']
70	+	elif line.startswith('logpath'):
71	+	line = 'logpath: %s\n'%DirStruct['logpath']
72	+	elif line.startswith('limits'):
73	+	line = 'limits: %s\n'%DirStruct['limitpath']
74	+	pathfile.write(line)
75	+	pathfile.close()
76	+
77	+	#copy config files
78	+	for item in configs:
79	+	shutil.copyfile(item,'%s/%s/%s'%(tagDir,opts.ftag,item.strip(en)))
80	+
81
30	–	en = opts.tag
31	–	#configs = ['config%s'%(en),'pathConfig%s'%(en)]
32	–	configs = ['#sconfig/general'%(en),'#sconfig/paths'%(en),'#sconfig/plots'%(en),'#sconfig/training'%(en),'#sconfig/datacards'%(en),'#sconfig/cuts'%(en)]
33	–
82		print configs
83		config = BetterConfigParser()
84		config.read(configs)
85	<	btagLibrary = config.get('BTagReshaping','library')
86	<	submitDir = os.getcwd()
87	<	os.chdir(os.path.dirname(btagLibrary))
88	<	if not os.path.exists(btagLibrary):
89	<	ROOT.gROOT.LoadMacro('%s+'%btagLibrary.replace('_h.so','.h'))
90	<	shutil.copyfile(os.path.basename(btagLibrary),'/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)))
91	<	shutil.copyfile('/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)),btagLibrary)
92	<	os.chdir(submitDir)
85	>
86	>	def dump_config(configs,output_file):
87	>	"""
88	>	Dump all the configs in a output file
89	>	Args:
90	>	output_file: the file where the log will be dumped
91	>	configs: list of files (string) to be dumped
92	>	Returns:
93	>	nothing
94	>	"""
95	>	outf = open(output_file,'w')
96	>	for i in configs:
97	>	try:
98	>	f=open(i,'r')
99	>	outf.write(f.read())
100	>	except: print '@WARNING: Config' + i + ' not found. It will not be used.'
101	>
102	>	def compile_macro(config,macro):
103	>	"""
104	>	Creates the library from a macro using CINT compiling it in scratch to avoid
105	>	problems with the linking in the working nodes.
106	>	Args:
107	>	config: configuration file where the macro path is specified
108	>	macro: macro name to be compiled
109	>	Returns:
110	>	nothing
111	>	"""
112	>	submitDir = os.getcwd()
113	>	_macro=macro+'.h'
114	>	library = config.get(macro,'library')
115	>	libDir=os.path.dirname(library)
116	>	os.chdir(libDir)
117	>	if not os.path.exists(library):
118	>	print '@INFO: Compiling ' + _macro
119	>	scratchDir='/scratch/%s/'%(getpass.getuser())
120	>	shutil.copyfile(libDir+'/'+_macro,'/scratch/%s/%s'%(getpass.getuser(),_macro))
121	>	os.chdir(scratchDir)
122	>	ROOT.gROOT.ProcessLine('.L %s+'%(scratchDir+_macro))
123	>	shutil.copyfile('/scratch/%s/%s'%(getpass.getuser(),os.path.basename(library)),library)
124	>	os.chdir(submitDir)
125	>
126	>	compile_macro(config,'BTagReshaping')
127	>	compile_macro(config,'VHbbNameSpace')
128	>
129		logPath = config.get("Directories","logpath")
130	+	logo = open('%s/data/submit.txt' %config.get('Directories','vhbbpath')).readlines()
131	+	counter = 0
132	+
133		#check if the logPath exist. If not exit
134		if( not os.path.isdir(logPath) ):
135	<	print 'ERROR: ' + logPath + ': dir not found.'
136	<	print 'ERROR: Create it before submitting '
137	<	print 'Exit'
138	<	sys.exit(-1)
135	>	print '@ERROR : ' + logPath + ': dir not found.'
136	>	print '@ERROR : Create it before submitting '
137	>	print 'Exit'
138	>	sys.exit(-1)
139	>
140
141	<	repDict = {'en':en,'logpath':logPath,'job':'','task':opts.task,'queue': 'all.q'}
141	>	repDict = {'en':en,'logpath':logPath,'job':'','task':opts.task,'queue': 'all.q','timestamp':timestamp,'additional':'','job_id':''}
142		def submit(job,repDict):
143	<	repDict['job'] = job
144	<	command = 'qsub -V -cwd -q %(queue)s -N %(job)s_%(en)s%(task)s -o %(logpath)s/%(job)s_%(en)s_%(task)s.out -e %(logpath)s/%(job)s_%(en)s_%(task)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task
145	<	print command
146	<	subprocess.call([command], shell=True)
143	>	global counter
144	>	repDict['job'] = job
145	>	nJob = counter % len(logo)
146	>	counter += 1
147	>	if opts.philipp_love_progress_bars:
148	>	repDict['name'] = '"%s"' %logo[nJob].strip()
149	>	else:
150	>	repDict['name'] = '%(job)s_%(en)s%(task)s' %repDict
151	>	command = 'qsub -V -cwd -q %(queue)s -l h_vmem=6G -N %(name)s -o %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.out -e %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task + ' ' + repDict['job_id'] + ' ' + repDict['additional']
152	>	print command
153	>	dump_config(configs,"%(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.config" %(repDict))
154	>	subprocess.call([command], shell=True)
155	>
156	>	if opts.task == 'train':
157	>	train_list = (config.get('MVALists','List_for_submitscript')).split(',')
158	>	print train_list
159	>	for item in train_list:
160	>	submit(item,repDict)
161	>
162
163		if opts.task == 'dc':
164		#DC_vars = config.items('Limit')
#	Line 68 | Line 171 | if opts.task == 'plot':
171		if not opts.task == 'prep':
172		path = config.get("Directories","samplepath")
173		samplesinfo = config.get("Directories","samplesinfo")
174	<	info = parse_info(samplesinfo,path)
174	>	info = ParseInfo(samplesinfo,path)
175
176		if opts.task == 'plot':
177		repDict['queue'] = 'all.q'
178		for item in Plot_vars:
179		submit(item,repDict)
180
181	<	elif opts.task == 'dc':
181	>	if opts.task == 'trainReg':
182		repDict['queue'] = 'all.q'
183	+	submit('trainReg',repDict)
184	+
185	+
186	+	elif opts.task == 'dc':
187	+	repDict['queue'] = 'short.q'
188		for item in DC_vars:
189		if 'ZH%s'%opts.mass in item:
190		submit(item,repDict)
#	Line 84 | Line 192 | elif opts.task == 'dc':
192		submit(item,repDict)
193
194		elif opts.task == 'prep':
195	<	submit('prepare',repDict)
195	>	if ( opts.samples == ""):
196	>	path = config.get("Directories","PREPin")
197	>	samplesinfo = config.get("Directories","samplesinfo")
198	>	info = ParseInfo(samplesinfo,path)
199	>	for job in info:
200	>	submit(job.name,repDict)
201
202	<	elif opts.task == 'eval' or opts.task == 'sys' or opts.task == 'syseval':
202	>	else:
203	>	for sample in samplesList:
204	>	submit(sample,repDict)
205	>	elif opts.task == 'sys' or opts.task == 'syseval':
206	>	path = config.get("Directories","SYSin")
207	>	samplesinfo = config.get("Directories","samplesinfo")
208	>	info = ParseInfo(samplesinfo,path)
209		if ( opts.samples == ""):
210		for job in info:
211	+	if (job.subsample):
212	+	continue #avoid multiple submissions form subsamples
213	+	# TO FIX FOR SPLITTED SAMPLE
214		submit(job.name,repDict)
215		else:
216		for sample in samplesList:
217		submit(sample,repDict)
218
219	<	os.system('qstat')
219	>	elif opts.task == 'eval':
220	>	path = config.get("Directories","MVAin")
221	>	samplesinfo = config.get("Directories","samplesinfo")
222	>	info = ParseInfo(samplesinfo,path)
223	>	if ( opts.samples == ""):
224	>	for job in info:
225	>	if (job.subsample):
226	>	continue #avoid multiple submissions from subsamples
227	>	if(info.checkSplittedSampleName(job.identifier)): # if multiple entries for one name  (splitted samples) use the identifier to submit
228	>	print '@INFO: Splitted samples: submit through identifier'
229	>	submit(job.identifier,repDict)
230	>	else: submit(job.name,repDict)
231	>	else:
232	>	for sample in samplesList:
233	>	submit(sample,repDict)
234	>
235	>
236	>	elif( opts.task == 'split' ):
237	>	path = config.get("Directories","SPLITin")
238	>	samplesinfo = config.get("Directories","samplesinfo")
239	>	repDict['job_id']=opts.nevents_split
240	>	info = ParseInfo(samplesinfo,path)
241	>	if ( opts.samples == "" ):
242	>	for job in info:
243	>	if (job.subsample): continue #avoid multiple submissions from subsamples
244	>	submit(job.name,repDict)
245	>	else:
246	>	for sample in samplesList:
247	>	submit(sample,repDict)
248	>
249	>	#BDT optimisation
250	>	elif opts.task == 'mva_opt':
251	>	total_number_of_steps=1
252	>	setting = ''
253	>	for par in (config.get('Optimisation','parameters').split(',')):
254	>	scan_par=eval(config.get('Optimisation',par))
255	>	setting+=par+'='+str(scan_par[0])+':'
256	>	if len(scan_par) > 1 and scan_par[2] != 0:
257	>	total_number_of_steps+=scan_par[2]
258	>	setting=setting[:-1] # eliminate last column at the end of the setting string
259	>	print setting
260	>	repDict['additional']=setting
261	>	repDict['job_id']=config.get('Optimisation','training')
262	>	submit('OPT_main_set',repDict)
263	>	main_setting=setting
264	>
265	>	#Scanning all the parameters found in the training config in the Optimisation sector
266	>	for par in (config.get('Optimisation','parameters').split(',')):
267	>	scan_par=eval(config.get('Optimisation',par))
268	>	print par
269	>	if len(scan_par) > 1 and scan_par[2] != 0:
270	>	for step in range(scan_par[2]):
271	>	value = (scan_par[0])+((1+step)*(scan_par[1]-scan_par[0])/scan_par[2])
272	>	print value
273	>	setting=re.sub(par+'.*?:',par+'='+str(value)+':',main_setting)
274	>	repDict['additional']=setting
275	>	#                               repDict['job_id']=config.get('Optimisation','training')
276	>	submit('OPT_'+par+str(value),repDict)
277	>	#                               submit(config.get('Optimisation','training'),repDict)
278	>	print setting
279	>
280	>
281	>	os.system('qstat')
282	>	if (opts.philipp_love_progress_bars):
283	>	os.system('./qstat.py')

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines