ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/UserCode/VHbb/python/submitThem.py

Comparing UserCode/VHbb/python/submitThem.py (file contents):
Revision 1.3 by nmohr, Mon Sep 17 19:06:15 2012 UTC vs.
Revision 1.30 by bortigno, Thu Feb 14 13:14:32 2013 UTC

#	Line 1 | Line 1
1		#! /usr/bin/env python
2	<	import os,time,subprocess
3	<	energy='8TeV'
4	<	#energy='7TeV'
2	>	from optparse import OptionParser
3	>	import sys
4	>	import time
5	>	import os
6	>	import shutil
7
8	<	def submit(job,en):
9	<	command = 'qsub -q all.q -N %s_%s -o /shome/nmohr/VHbbAnalysis/LOG/%s.out -e /shome/nmohr/VHbbAnalysis/LOG/%s.err runAll.sh %s %s' %(job,en,job,job,job,en)
8	>	parser = OptionParser()
9	>	parser.add_option("-T", "--tag", dest="tag", default="",
10	>	help="Tag to run the analysis with, example '8TeV' uses config8TeV and pathConfig8TeV to run the analysis")
11	>	parser.add_option("-J", "--task", dest="task", default="",
12	>	help="Task to be done, i.e. 'dc' for Datacards, 'prep' for preparation of Trees, 'plot' to produce plots or 'eval' to write the MVA output or 'sys' to write regression and systematics (or 'syseval' for both). ")
13	>	parser.add_option("-M", "--mass", dest="mass", default="125",
14	>	help="Mass for DC or Plots, 110...135")
15	>	parser.add_option("-S","--samples",dest="samples",default="",
16	>	help="samples you want to run on")
17	>	parser.add_option("-F", "--folderTag", dest="ftag", default="",
18	>	help="Creats a new folder structure for outputs or uses an existing one with the given name")
19	>	parser.add_option("-N", "--number-of-events", dest="nevents_split", default=100000,
20	>	help="Number of events per file when splitting.")
21	>	parser.add_option("-P", "--philipp-love-progress-bars", dest="philipp_love_progress_bars", default=False,
22	>	help="If you share the love of Philipp...")
23	>
24	>	(opts, args) = parser.parse_args(sys.argv)
25	>
26	>	import os,shutil,pickle,subprocess,ROOT,re
27	>	ROOT.gROOT.SetBatch(True)
28	>	from myutils import BetterConfigParser, Sample, ParseInfo
29	>	import getpass
30	>
31	>	if opts.tag == "":
32	>	print "Please provide tag to run the analysis with, example '-T 8TeV' uses config8TeV and pathConfig8TeV to run the analysis."
33	>	sys.exit(123)
34	>
35	>	if opts.task == "":
36	>	print "Please provide a task.\n-J prep:\tpreparation of Trees\n-J sys:\t\twrite regression and systematics\n-J eval:\tcreate MVA output\n-J plot:\tproduce Plots\n-J dc:\t\twrite workspaces and datacards"
37	>	sys.exit(123)
38	>
39	>
40	>	en = opts.tag
41	>
42	>	#create the list with the samples to run over
43	>	samplesList=opts.samples.split(",")
44	>
45	>	timestamp = time.asctime().replace(' ','_').replace(':','-')
46	>
47	>	configs = ['%sconfig/general'%(en),'%sconfig/paths'%(en),'%sconfig/plots'%(en),'%sconfig/training'%(en),'%sconfig/datacards'%(en),'%sconfig/cuts'%(en)]
48	>
49	>	pathconfig = BetterConfigParser()
50	>	pathconfig.read('%sconfig/paths'%(en))
51	>
52	>	if not opts.ftag == '':
53	>	tagDir = pathconfig.get('Directories','tagDir')
54	>	DirStruct={'tagDir':tagDir,'ftagdir':'%s/%s/'%(tagDir,opts.ftag),'logpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Logs'),'plotpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Plots'),'limitpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Limits'),'confpath':'%s/%s/%s/'%(tagDir,opts.ftag,'config') }
55	>
56	>	for keys in ['tagDir','ftagdir','logpath','plotpath','limitpath','confpath']:
57	>	try:
58	>	os.stat(DirStruct[keys])
59	>	except:
60	>	os.mkdir(DirStruct[keys])
61	>
62	>	pathfile = open('%sconfig/paths'%(en))
63	>	buffer = pathfile.readlines()
64	>	pathfile.close()
65	>	os.rename('%sconfig/paths'%(en),'%sconfig/paths.bkp'%(en))
66	>	pathfile = open('%sconfig/paths'%(en),'w')
67	>	for line in buffer:
68	>	if line.startswith('plotpath'):
69	>	line = 'plotpath: %s\n'%DirStruct['plotpath']
70	>	elif line.startswith('logpath'):
71	>	line = 'logpath: %s\n'%DirStruct['logpath']
72	>	elif line.startswith('limits'):
73	>	line = 'limits: %s\n'%DirStruct['limitpath']
74	>	pathfile.write(line)
75	>	pathfile.close()
76	>
77	>	#copy config files
78	>	for item in configs:
79	>	shutil.copyfile(item,'%s/%s/%s'%(tagDir,opts.ftag,item.strip(en)))
80	>
81	>
82	>	print configs
83	>	config = BetterConfigParser()
84	>	config.read(configs)
85	>
86	>	btagLibrary = config.get('BTagReshaping','library')
87	>	submitDir = os.getcwd()
88	>	os.chdir(os.path.dirname(btagLibrary))
89	>	if not os.path.exists(btagLibrary):
90	>	ROOT.gROOT.LoadMacro('%s+'%btagLibrary.replace('_h.so','.h'))
91	>	shutil.copyfile(os.path.basename(btagLibrary),'/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)))
92	>	shutil.copyfile('/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)),btagLibrary)
93	>	os.chdir(submitDir)
94	>	logPath = config.get("Directories","logpath")
95	>	#check if the logPath exist. If not exit
96	>	if( not os.path.isdir(logPath) ):
97	>	print 'ERROR: ' + logPath + ': dir not found.'
98	>	print 'ERROR: Create it before submitting '
99	>	print 'Exit'
100	>	sys.exit(-1)
101	>
102	>	repDict = {'en':en,'logpath':logPath,'job':'','task':opts.task,'queue': 'all.q','timestamp':timestamp,'additional':'','job_id':''}
103	>	def submit(job,repDict):
104	>	repDict['job'] = job
105	>	command = 'qsub -V -cwd -q %(queue)s -l h_vmem=6G -N %(job)s_%(en)s%(task)s -o %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.out -e %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task + ' ' + repDict['job_id'] + ' ' + repDict['additional']
106		print command
107		subprocess.call([command], shell=True)
108
109	<	theJobs = ['ZH110','ZH125','ZH120','DY','ZH115','ZH130','ZH135','ZZ','DY120','Zmm','ST_s','TT','Zee','STbar_s','STbar_t','WZ','WW','STbar_tW','ST_tW']
110	<	if energy=='8TeV':
111	<	theJobs = ['ZH110','ZH125','ZH120','DY','DY5070','DY70100','DY100','ZH115','ZH130','ZH135','ZZ','DY120','Zmm','ST_s','TT','Zee','STbar_s','STbar_t','WZ','WW','STbar_tW','ST_tW']
109	>	if opts.task == 'train':
110	>	train_list = (config.get('MVALists','List_for_submitscript')).split(',')
111	>	print train_list
112	>	for item in train_list:
113	>	submit(item,repDict)
114	>
115	>
116	>	if opts.task == 'dc':
117	>	#DC_vars = config.items('Limit')
118	>	DC_vars= (config.get('LimitGeneral','List')).split(',')
119	>	print DC_vars
120	>
121	>	if opts.task == 'plot':
122	>	Plot_vars= (config.get('Plot_general','List')).split(',')
123	>
124	>	if not opts.task == 'prep':
125	>	path = config.get("Directories","samplepath")
126	>	samplesinfo = config.get("Directories","samplesinfo")
127	>	info = ParseInfo(samplesinfo,path)
128	>
129	>	if opts.task == 'plot':
130	>	repDict['queue'] = 'all.q'
131	>	for item in Plot_vars:
132	>	submit(item,repDict)
133	>
134	>
135	>	elif opts.task == 'dc':
136	>	repDict['queue'] = 'all.q'
137	>	for item in DC_vars:
138	>	if 'ZH%s'%opts.mass in item:
139	>	submit(item,repDict)
140	>	elif 'ZH' in item and opts.mass == 'all':
141	>	submit(item,repDict)
142	>
143	>	elif opts.task == 'prep':
144	>	submit('prepare',repDict)
145	>
146	>	elif opts.task == 'sys' or opts.task == 'syseval':
147	>	path = config.get("Directories","SYSin")
148	>	samplesinfo = config.get("Directories","samplesinfo")
149	>	info = ParseInfo(samplesinfo,path)
150	>	if ( opts.samples == ""):
151	>	for job in info:
152	>	if (job.subsample): continue
153	>	submit(job.name,repDict)
154	>	else:
155	>	for sample in samplesList:
156	>	submit(sample,repDict)
157	>
158	>	elif opts.task == 'eval':
159	>	path = config.get("Directories","MVAin")
160	>	samplesinfo = config.get("Directories","samplesinfo")
161	>	info = ParseInfo(samplesinfo,path)
162	>	if ( opts.samples == ""):
163	>	for job in info:
164	>	if (job.subsample): continue
165	>	submit(job.name,repDict)
166	>	else:
167	>	for sample in samplesList:
168	>	submit(sample,repDict)
169	>
170	>
171	>	elif( opts.task == 'split' ):
172	>	path = config.get("Directories","SPLITin")
173	>	samplesinfo = config.get("Directories","samplesinfo")
174	>	repDict['additional']=opts.nevents_split
175	>	info = ParseInfo(samplesinfo,path)
176	>	if ( opts.samples == ""):
177	>	for job in info:
178	>	if (job.subsample): continue
179	>	submit(job.name,repDict)
180	>	else:
181	>	for sample in samplesList:
182	>	submit(sample,repDict)
183	>
184	>	#BDT optimisation
185	>	elif opts.task == 'mva_opt':
186	>	total_number_of_steps=1
187	>	setting = ''
188	>	for par in (config.get('Optimisation','parameters').split(',')):
189	>	scan_par=eval(config.get('Optimisation',par))
190	>	setting+=par+'='+str(scan_par[0])+':'
191	>	if len(scan_par) > 1 and scan_par[2] != 0:
192	>	total_number_of_steps+=scan_par[2]
193	>	setting=setting[:-1] # eliminate last column at the end of the setting string
194	>	print setting
195	>	repDict['additional']=setting
196	>	repDict['job_id']=config.get('Optimisation','training')
197	>	submit('OPT_main_set',repDict)
198	>	main_setting=setting
199	>
200	>	#Scanning all the parameters found in the training config in the Optimisation sector
201	>	for par in (config.get('Optimisation','parameters').split(',')):
202	>	scan_par=eval(config.get('Optimisation',par))
203	>	print par
204	>	if len(scan_par) > 1 and scan_par[2] != 0:
205	>	for step in range(scan_par[2]):
206	>	value = (scan_par[0])+((1+step)*(scan_par[1]-scan_par[0])/scan_par[2])
207	>	print value
208	>	setting=re.sub(par+'.*?:',par+'='+str(value)+':',main_setting)
209	>	repDict['additional']=setting
210	>	#                               repDict['job_id']=config.get('Optimisation','training')
211	>	submit('OPT_'+par+str(value),repDict)
212	>	#                               submit(config.get('Optimisation','training'),repDict)
213	>	print setting
214
215
216	<	for job in theJobs:
217	<	submit(job,energy)
218	<	#time.sleep(10)
216	>	os.system('qstat')
217	>	if (opts.philipp_love_progress_bars):
218	>	os.system('./qstat.py')

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines