ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/UserCode/VHbb/python/submitThem.py

Comparing UserCode/VHbb/python/submitThem.py (file contents):
Revision 1.12 by peller, Mon Oct 1 09:16:34 2012 UTC vs.
Revision 1.34 by nmohr, Thu Feb 28 16:59:09 2013 UTC

#	Line 1 | Line 1
1		#! /usr/bin/env python
2	–	import os,shutil,sys,pickle,subprocess,ROOT
2		from optparse import OptionParser
3	<	from BetterConfigParser import BetterConfigParser
4	<	from samplesclass import sample
5	<	import getpass
3	>	import sys
4	>	import time
5	>	import os
6	>	import shutil
7
8		parser = OptionParser()
9		parser.add_option("-T", "--tag", dest="tag", default="",
10		help="Tag to run the analysis with, example '8TeV' uses config8TeV and pathConfig8TeV to run the analysis")
11		parser.add_option("-J", "--task", dest="task", default="",
12	<	help="Task to be done, i.e. 'dc' for Datacards, 'prep' for preparation of Trees, 'plot' to produce plots or 'eval' to write the MVA output or 'sys' to write regression and systematics. ")
12	>	help="Task to be done, i.e. 'dc' for Datacards, 'prep' for preparation of Trees, 'plot' to produce plots or 'eval' to write the MVA output or 'sys' to write regression and systematics (or 'syseval' for both). ")
13		parser.add_option("-M", "--mass", dest="mass", default="125",
14		help="Mass for DC or Plots, 110...135")
15		parser.add_option("-S","--samples",dest="samples",default="",
16		help="samples you want to run on")
17	<
17	>	parser.add_option("-F", "--folderTag", dest="ftag", default="",
18	>	help="Creats a new folder structure for outputs or uses an existing one with the given name")
19	>	parser.add_option("-N", "--number-of-events", dest="nevents_split", default=100000,
20	>	help="Number of events per file when splitting.")
21	>	parser.add_option("-P", "--philipp-love-progress-bars", dest="philipp_love_progress_bars", default=False,
22	>	help="If you share the love of Philipp...")
23
24		(opts, args) = parser.parse_args(sys.argv)
25	+
26	+	import os,shutil,pickle,subprocess,ROOT,re
27	+	ROOT.gROOT.SetBatch(True)
28	+	from myutils import BetterConfigParser, Sample, ParseInfo, sample_parser
29	+	import getpass
30	+
31		if opts.tag == "":
32		print "Please provide tag to run the analysis with, example '-T 8TeV' uses config8TeV and pathConfig8TeV to run the analysis."
33		sys.exit(123)
#	Line 25 | Line 36 | if opts.task == "":
36		print "Please provide a task.\n-J prep:\tpreparation of Trees\n-J sys:\t\twrite regression and systematics\n-J eval:\tcreate MVA output\n-J plot:\tproduce Plots\n-J dc:\t\twrite workspaces and datacards"
37		sys.exit(123)
38
39	+
40	+	en = opts.tag
41	+
42		#create the list with the samples to run over
43		samplesList=opts.samples.split(",")
44
45	<	en = opts.tag
46	<	configs = ['config%s'%(en),'pathConfig%s'%(en)]
47	<
45	>	timestamp = time.asctime().replace(' ','_').replace(':','-')
46	>
47	>	configs = ['%sconfig/general'%(en),'%sconfig/paths'%(en),'%sconfig/plots'%(en),'%sconfig/training'%(en),'%sconfig/datacards'%(en),'%sconfig/cuts'%(en)]
48	>
49	>	pathconfig = BetterConfigParser()
50	>	pathconfig.read('%sconfig/paths'%(en))
51	>
52	>	if not opts.ftag == '':
53	>	tagDir = pathconfig.get('Directories','tagDir')
54	>	DirStruct={'tagDir':tagDir,'ftagdir':'%s/%s/'%(tagDir,opts.ftag),'logpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Logs'),'plotpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Plots'),'limitpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Limits'),'confpath':'%s/%s/%s/'%(tagDir,opts.ftag,'config') }
55	>
56	>	for keys in ['tagDir','ftagdir','logpath','plotpath','limitpath','confpath']:
57	>	try:
58	>	os.stat(DirStruct[keys])
59	>	except:
60	>	os.mkdir(DirStruct[keys])
61	>
62	>	pathfile = open('%sconfig/paths'%(en))
63	>	buffer = pathfile.readlines()
64	>	pathfile.close()
65	>	os.rename('%sconfig/paths'%(en),'%sconfig/paths.bkp'%(en))
66	>	pathfile = open('%sconfig/paths'%(en),'w')
67	>	for line in buffer:
68	>	if line.startswith('plotpath'):
69	>	line = 'plotpath: %s\n'%DirStruct['plotpath']
70	>	elif line.startswith('logpath'):
71	>	line = 'logpath: %s\n'%DirStruct['logpath']
72	>	elif line.startswith('limits'):
73	>	line = 'limits: %s\n'%DirStruct['limitpath']
74	>	pathfile.write(line)
75	>	pathfile.close()
76	>
77	>	#copy config files
78	>	for item in configs:
79	>	shutil.copyfile(item,'%s/%s/%s'%(tagDir,opts.ftag,item.strip(en)))
80	>
81	>
82		print configs
83		config = BetterConfigParser()
84		config.read(configs)
85	+
86		btagLibrary = config.get('BTagReshaping','library')
87		submitDir = os.getcwd()
88		os.chdir(os.path.dirname(btagLibrary))
#	Line 43 | Line 92 | shutil.copyfile(os.path.basename(btagLib
92		shutil.copyfile('/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)),btagLibrary)
93		os.chdir(submitDir)
94		logPath = config.get("Directories","logpath")
95	<	repDict = {'en':en,'logpath':logPath,'job':''}
95	>	logo = open('%s/data/submit.txt' %config.get('Directories','vhbbpath')).readlines()
96	>	counter = 0
97	>
98	>	#check if the logPath exist. If not exit
99	>	if( not os.path.isdir(logPath) ):
100	>	print 'ERROR: ' + logPath + ': dir not found.'
101	>	print 'ERROR: Create it before submitting '
102	>	print 'Exit'
103	>	sys.exit(-1)
104	>
105	>
106	>	repDict = {'en':en,'logpath':logPath,'job':'','task':opts.task,'queue': 'all.q','timestamp':timestamp,'additional':'','job_id':''}
107		def submit(job,repDict):
108	<	repDict['job'] = job
109	<	command = 'qsub -V -cwd -q all.q -N %(job)s_%(en)s -o %(logpath)s/%(job)s_%(en)s.out -e %(logpath)s/%(job)s_%(en)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task
110	<	print command
111	<	subprocess.call([command], shell=True)
108	>	global counter
109	>	repDict['job'] = job
110	>	nJob = counter % len(logo)
111	>	counter += 1
112	>	if opts.philipp_love_progress_bars:
113	>	repDict['name'] = '"%s"' %logo[nJob].strip()
114	>	else:
115	>	repDict['name'] = '%(job)s_%(en)s%(task)s' %repDict
116	>	command = 'qsub -V -cwd -q %(queue)s -l h_vmem=6G -N %(name)s -o %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.out -e %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task + ' ' + repDict['job_id'] + ' ' + repDict['additional']
117	>	print command
118	>	subprocess.call([command], shell=True)
119	>
120	>	if opts.task == 'train':
121	>	train_list = (config.get('MVALists','List_for_submitscript')).split(',')
122	>	print train_list
123	>	for item in train_list:
124	>	submit(item,repDict)
125	>
126
127		if opts.task == 'dc':
128	<	DC_vars = config.items('Limit')
128	>	#DC_vars = config.items('Limit')
129	>	DC_vars= (config.get('LimitGeneral','List')).split(',')
130	>	print DC_vars
131	>
132		if opts.task == 'plot':
133	<	Plot_vars= config.items('Plot')
133	>	Plot_vars= (config.get('Plot_general','List')).split(',')
134
135		if not opts.task == 'prep':
136		path = config.get("Directories","samplepath")
137	<	infofile = open(path+'/env/samples.info','r')
138	<	info = pickle.load(infofile)
62	<	infofile.close()
63	<
137	>	samplesinfo = config.get("Directories","samplesinfo")
138	>	info = ParseInfo(samplesinfo,path)
139
140		if opts.task == 'plot':
141	+	repDict['queue'] = 'all.q'
142		for item in Plot_vars:
143	<	if 'ZH%s'%opts.mass in item[0]:
144	<	submit(item[0],repDict)
69	<	elif opts.mass == '' and 'ZH' in item[0]:
70	<	submit(item[0],repDict)
143	>	submit(item,repDict)
144	>
145
146		elif opts.task == 'dc':
147	+	repDict['queue'] = 'short.q'
148		for item in DC_vars:
149	<	if 'ZH%s'%opts.mass in item[0] and opts.tag in item[0]:
150	<	submit(item[0],repDict)
151	<	elif 'ZH' in item[0] and opts.tag in item[0] and opts.mass == '*':
152	<	submit(item[0],repDict)
149	>	if 'ZH%s'%opts.mass in item:
150	>	submit(item,repDict)
151	>	elif 'ZH' in item and opts.mass == 'all':
152	>	submit(item,repDict)
153	>
154		elif opts.task == 'prep':
155	<	submit('prepare',repDict)
155	>	if ( opts.samples == ""):
156	>	path = config.get("Directories","PREPin")
157	>	samplesinfo = config.get("Directories","samplesinfo")
158	>	info = ParseInfo(samplesinfo,path)
159	>	for job in info:
160	>	submit(job.name,repDict)
161
162	<	elif opts.task == 'eval' or opts.task == 'sys':
162	>	else:
163	>	for sample in samplesList:
164	>	submit(sample,repDict)
165	>	elif opts.task == 'sys' or opts.task == 'syseval':
166	>	path = config.get("Directories","SYSin")
167	>	samplesinfo = config.get("Directories","samplesinfo")
168	>	info = ParseInfo(samplesinfo,path)
169		if ( opts.samples == ""):
170		for job in info:
171	+	if (job.subsample):
172	+	continue #avoid multiple submissions form subsamples
173	+	# TO FIX FOR SPLITTED SAMPLE
174		submit(job.name,repDict)
175		else:
176		for sample in samplesList:
177		submit(sample,repDict)
178	<
178	>
179	>	elif opts.task == 'eval':
180	>	path = config.get("Directories","MVAin")
181	>	samplesinfo = config.get("Directories","samplesinfo")
182	>	info = ParseInfo(samplesinfo,path)
183	>	if ( opts.samples == ""):
184	>	for job in info:
185	>	if (job.subsample):
186	>	continue #avoid multiple submissions from subsamples
187	>	if(info.checkSplittedSampleName(job.identifier)): # if multiple entries for one name  (splitted samples) use the identifier to submit
188	>	print '@INFO: Splitted samples: submit through identifier'
189	>	submit(job.identifier,repDict)
190	>	else: submit(job.name,repDict)
191	>	else:
192	>	for sample in samplesList:
193	>	submit(sample,repDict)
194	>
195	>
196	>	elif( opts.task == 'split' ):
197	>	path = config.get("Directories","SPLITin")
198	>	samplesinfo = config.get("Directories","samplesinfo")
199	>	repDict['job_id']=opts.nevents_split
200	>	info = ParseInfo(samplesinfo,path)
201	>	if ( opts.samples == "" ):
202	>	for job in info:
203	>	if (job.subsample): continue #avoid multiple submissions from subsamples
204	>	submit(job.name,repDict)
205	>	else:
206	>	for sample in samplesList:
207	>	submit(sample,repDict)
208	>
209	>	#BDT optimisation
210	>	elif opts.task == 'mva_opt':
211	>	total_number_of_steps=1
212	>	setting = ''
213	>	for par in (config.get('Optimisation','parameters').split(',')):
214	>	scan_par=eval(config.get('Optimisation',par))
215	>	setting+=par+'='+str(scan_par[0])+':'
216	>	if len(scan_par) > 1 and scan_par[2] != 0:
217	>	total_number_of_steps+=scan_par[2]
218	>	setting=setting[:-1] # eliminate last column at the end of the setting string
219	>	print setting
220	>	repDict['additional']=setting
221	>	repDict['job_id']=config.get('Optimisation','training')
222	>	submit('OPT_main_set',repDict)
223	>	main_setting=setting
224	>
225	>	#Scanning all the parameters found in the training config in the Optimisation sector
226	>	for par in (config.get('Optimisation','parameters').split(',')):
227	>	scan_par=eval(config.get('Optimisation',par))
228	>	print par
229	>	if len(scan_par) > 1 and scan_par[2] != 0:
230	>	for step in range(scan_par[2]):
231	>	value = (scan_par[0])+((1+step)*(scan_par[1]-scan_par[0])/scan_par[2])
232	>	print value
233	>	setting=re.sub(par+'.*?:',par+'='+str(value)+':',main_setting)
234	>	repDict['additional']=setting
235	>	#                               repDict['job_id']=config.get('Optimisation','training')
236	>	submit('OPT_'+par+str(value),repDict)
237	>	#                               submit(config.get('Optimisation','training'),repDict)
238	>	print setting
239	>
240	>
241	>	os.system('qstat')
242	>	if (opts.philipp_love_progress_bars):
243	>	os.system('./qstat.py')

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines