ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/UserCode/VHbb/python/submitThem.py

Comparing UserCode/VHbb/python/submitThem.py (file contents):
Revision 1.10 by bortigno, Sun Sep 30 17:36:39 2012 UTC vs.
Revision 1.36 by nmohr, Fri Mar 22 15:19:28 2013 UTC

#	Line 1 | Line 1
1		#! /usr/bin/env python
2	–	import os,shutil,sys,pickle,subprocess,ROOT
2		from optparse import OptionParser
3	<	from BetterConfigParser import BetterConfigParser
4	<	from samplesclass import sample
5	<	import getpass
3	>	import sys
4	>	import time
5	>	import os
6	>	import shutil
7
8		parser = OptionParser()
9		parser.add_option("-T", "--tag", dest="tag", default="",
10		help="Tag to run the analysis with, example '8TeV' uses config8TeV and pathConfig8TeV to run the analysis")
11		parser.add_option("-J", "--task", dest="task", default="",
12	<	help="Task to be done, i.e. 'dc' for Datacards, 'prep' for preparation of Trees, 'plot' to produce plots or 'eval' to write the MVA output or 'sys' to write regression and systematics. ")
12	>	help="Task to be done, i.e. 'dc' for Datacards, 'prep' for preparation of Trees, 'plot' to produce plots or 'eval' to write the MVA output or 'sys' to write regression and systematics (or 'syseval' for both). ")
13		parser.add_option("-M", "--mass", dest="mass", default="125",
14		help="Mass for DC or Plots, 110...135")
15		parser.add_option("-S","--samples",dest="samples",default="",
16		help="samples you want to run on")
17	<
17	>	parser.add_option("-F", "--folderTag", dest="ftag", default="",
18	>	help="Creats a new folder structure for outputs or uses an existing one with the given name")
19	>	parser.add_option("-N", "--number-of-events", dest="nevents_split", default=100000,
20	>	help="Number of events per file when splitting.")
21	>	parser.add_option("-P", "--philipp-love-progress-bars", dest="philipp_love_progress_bars", default=False,
22	>	help="If you share the love of Philipp...")
23
24		(opts, args) = parser.parse_args(sys.argv)
25	+
26	+	import os,shutil,pickle,subprocess,ROOT,re
27	+	ROOT.gROOT.SetBatch(True)
28	+	from myutils import BetterConfigParser, Sample, ParseInfo, sample_parser
29	+	import getpass
30	+
31		if opts.tag == "":
32		print "Please provide tag to run the analysis with, example '-T 8TeV' uses config8TeV and pathConfig8TeV to run the analysis."
33		sys.exit(123)
#	Line 25 | Line 36 | if opts.task == "":
36		print "Please provide a task.\n-J prep:\tpreparation of Trees\n-J sys:\t\twrite regression and systematics\n-J eval:\tcreate MVA output\n-J plot:\tproduce Plots\n-J dc:\t\twrite workspaces and datacards"
37		sys.exit(123)
38
28	–	#create the dictionary with the samples to run over
29	–	samplesDict=opts.samples.split(",")
39
40		en = opts.tag
41	<	configs = ['config%s'%(en),'pathConfig%s'%(en)]
42	<
41	>
42	>	#create the list with the samples to run over
43	>	samplesList=opts.samples.split(",")
44	>	timestamp = time.asctime().replace(' ','_').replace(':','-')
45	>
46	>	# the list of the config is taken from the path config
47	>	pathconfig = BetterConfigParser()
48	>	pathconfig.read('%sconfig/paths'%(en))
49	>	_configs = pathconfig.get('Configuration','List').split(" ")
50	>	configs = [ '%sconfig/'%(en) + c for c in _configs  ]
51	>
52	>	if not opts.ftag == '':
53	>	tagDir = pathconfig.get('Directories','tagDir')
54	>	DirStruct={'tagDir':tagDir,'ftagdir':'%s/%s/'%(tagDir,opts.ftag),'logpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Logs'),'plotpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Plots'),'limitpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Limits'),'confpath':'%s/%s/%s/'%(tagDir,opts.ftag,'config') }
55	>
56	>	for keys in ['tagDir','ftagdir','logpath','plotpath','limitpath','confpath']:
57	>	try:
58	>	os.stat(DirStruct[keys])
59	>	except:
60	>	os.mkdir(DirStruct[keys])
61	>
62	>	pathfile = open('%sconfig/paths'%(en))
63	>	buffer = pathfile.readlines()
64	>	pathfile.close()
65	>	os.rename('%sconfig/paths'%(en),'%sconfig/paths.bkp'%(en))
66	>	pathfile = open('%sconfig/paths'%(en),'w')
67	>	for line in buffer:
68	>	if line.startswith('plotpath'):
69	>	line = 'plotpath: %s\n'%DirStruct['plotpath']
70	>	elif line.startswith('logpath'):
71	>	line = 'logpath: %s\n'%DirStruct['logpath']
72	>	elif line.startswith('limits'):
73	>	line = 'limits: %s\n'%DirStruct['limitpath']
74	>	pathfile.write(line)
75	>	pathfile.close()
76	>
77	>	#copy config files
78	>	for item in configs:
79	>	shutil.copyfile(item,'%s/%s/%s'%(tagDir,opts.ftag,item.strip(en)))
80	>
81	>
82		print configs
83		config = BetterConfigParser()
84		config.read(configs)
85	<	btagLibrary = config.get('BTagReshaping','library')
86	<	submitDir = os.getcwd()
87	<	os.chdir(os.path.dirname(btagLibrary))
88	<	if not os.path.exists(btagLibrary):
89	<	ROOT.gROOT.LoadMacro('%s+'%btagLibrary.replace('_h.so','.h'))
90	<	shutil.copyfile(os.path.basename(btagLibrary),'/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)))
91	<	shutil.copyfile('/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)),btagLibrary)
92	<	os.chdir(submitDir)
85	>
86	>
87	>	def compile_macro(config,macro):
88	>	"""
89	>	Creates the library from a macro using CINT compiling it in scratch to avoid
90	>	problems with the linking in the working nodes.
91	>	Args:
92	>	config: configuration file where the macro path is specified
93	>	macro: macro name to be compiled
94	>	Returns:
95	>	nothing
96	>	"""
97	>	submitDir = os.getcwd()
98	>	_macro=macro+'.h'
99	>	library = config.get(macro,'library')
100	>	libDir=os.path.dirname(library)
101	>	os.chdir(libDir)
102	>	if not os.path.exists(library):
103	>	print '@INFO: Compiling ' + _macro
104	>	scratchDir='/scratch/%s/'%(getpass.getuser())
105	>	shutil.copyfile(libDir+'/'+_macro,'/scratch/%s/%s'%(getpass.getuser(),_macro))
106	>	os.chdir(scratchDir)
107	>	ROOT.gROOT.ProcessLine('.L %s+'%(scratchDir+_macro))
108	>	shutil.copyfile('/scratch/%s/%s'%(getpass.getuser(),os.path.basename(library)),library)
109	>	os.chdir(submitDir)
110	>
111	>	compile_macro(config,'BTagReshaping')
112	>	compile_macro(config,'VHbbNameSpace')
113	>
114		logPath = config.get("Directories","logpath")
115	<	repDict = {'en':en,'logpath':logPath,'job':''}
115	>	logo = open('%s/data/submit.txt' %config.get('Directories','vhbbpath')).readlines()
116	>	counter = 0
117	>
118	>	#check if the logPath exist. If not exit
119	>	if( not os.path.isdir(logPath) ):
120	>	print '@ERROR : ' + logPath + ': dir not found.'
121	>	print '@ERROR : Create it before submitting '
122	>	print 'Exit'
123	>	sys.exit(-1)
124	>
125	>
126	>	repDict = {'en':en,'logpath':logPath,'job':'','task':opts.task,'queue': 'all.q','timestamp':timestamp,'additional':'','job_id':''}
127		def submit(job,repDict):
128	<	repDict['job'] = job
129	<	command = 'qsub -V -cwd -q all.q -N %(job)s_%(en)s -o %(logpath)s/%(job)s_%(en)s.out -e %(logpath)s/%(job)s_%(en)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task
130	<	print command
131	<	subprocess.call([command], shell=True)
128	>	global counter
129	>	repDict['job'] = job
130	>	nJob = counter % len(logo)
131	>	counter += 1
132	>	if opts.philipp_love_progress_bars:
133	>	repDict['name'] = '"%s"' %logo[nJob].strip()
134	>	else:
135	>	repDict['name'] = '%(job)s_%(en)s%(task)s' %repDict
136	>	command = 'qsub -V -cwd -q %(queue)s -l h_vmem=6G -N %(name)s -o %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.out -e %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task + ' ' + repDict['job_id'] + ' ' + repDict['additional']
137	>	print command
138	>	subprocess.call([command], shell=True)
139	>
140	>	if opts.task == 'train':
141	>	train_list = (config.get('MVALists','List_for_submitscript')).split(',')
142	>	print train_list
143	>	for item in train_list:
144	>	submit(item,repDict)
145	>
146
147		if opts.task == 'dc':
148	<	DC_vars = config.items('Limit')
149	<	if opts.task == 'plot':
150	<	Plot_vars= config.items('Plot')
148	>	#DC_vars = config.items('Limit')
149	>	DC_vars= (config.get('LimitGeneral','List')).split(',')
150	>	print DC_vars
151
152	<	path = config.get("Directories","samplepath")
153	<	infofile = open(path+'/env/samples.info','r')
60	<	info = pickle.load(infofile)
61	<	infofile.close()
152	>	if opts.task == 'plot':
153	>	Plot_vars= (config.get('Plot_general','List')).split(',')
154
155	+	if not opts.task == 'prep':
156	+	path = config.get("Directories","samplepath")
157	+	samplesinfo = config.get("Directories","samplesinfo")
158	+	info = ParseInfo(samplesinfo,path)
159
160		if opts.task == 'plot':
161	+	repDict['queue'] = 'all.q'
162		for item in Plot_vars:
163	<	if 'ZH%s'%opts.mass in item[0]:
164	<	submit(item[0],repDict)
165	<	elif opts.mass == '' and 'ZH' in item[0]:
166	<	submit(item[0],repDict)
163	>	submit(item,repDict)
164	>
165	>	if opts.task == 'trainReg':
166	>	repDict['queue'] = 'all.q'
167	>	submit('trainReg',repDict)
168	>
169
170		elif opts.task == 'dc':
171	+	repDict['queue'] = 'short.q'
172		for item in DC_vars:
173	<	if 'ZH%s'%opts.mass in item[0] and opts.tag in item[0]:
174	<	submit(item[0],repDict)
175	<	elif 'ZH' in item[0] and opts.tag in item[0] and opts.mass == '*':
176	<	submit(item[0],repDict)
173	>	if 'ZH%s'%opts.mass in item:
174	>	submit(item,repDict)
175	>	elif 'ZH' in item and opts.mass == 'all':
176	>	submit(item,repDict)
177	>
178		elif opts.task == 'prep':
179	<	submit('prepare',repDict)
179	>	if ( opts.samples == ""):
180	>	path = config.get("Directories","PREPin")
181	>	samplesinfo = config.get("Directories","samplesinfo")
182	>	info = ParseInfo(samplesinfo,path)
183	>	for job in info:
184	>	submit(job.name,repDict)
185
186	<	elif opts.task == 'eval' or opts.task == 'sys':
186	>	else:
187	>	for sample in samplesList:
188	>	submit(sample,repDict)
189	>	elif opts.task == 'sys' or opts.task == 'syseval':
190	>	path = config.get("Directories","SYSin")
191	>	samplesinfo = config.get("Directories","samplesinfo")
192	>	info = ParseInfo(samplesinfo,path)
193		if ( opts.samples == ""):
194		for job in info:
195	+	if (job.subsample):
196	+	continue #avoid multiple submissions form subsamples
197	+	# TO FIX FOR SPLITTED SAMPLE
198		submit(job.name,repDict)
199		else:
200	<	for sample in samplesDict:
200	>	for sample in samplesList:
201		submit(sample,repDict)
202	<
202	>
203	>	elif opts.task == 'eval':
204	>	path = config.get("Directories","MVAin")
205	>	samplesinfo = config.get("Directories","samplesinfo")
206	>	info = ParseInfo(samplesinfo,path)
207	>	if ( opts.samples == ""):
208	>	for job in info:
209	>	if (job.subsample):
210	>	continue #avoid multiple submissions from subsamples
211	>	if(info.checkSplittedSampleName(job.identifier)): # if multiple entries for one name  (splitted samples) use the identifier to submit
212	>	print '@INFO: Splitted samples: submit through identifier'
213	>	submit(job.identifier,repDict)
214	>	else: submit(job.name,repDict)
215	>	else:
216	>	for sample in samplesList:
217	>	submit(sample,repDict)
218	>
219	>
220	>	elif( opts.task == 'split' ):
221	>	path = config.get("Directories","SPLITin")
222	>	samplesinfo = config.get("Directories","samplesinfo")
223	>	repDict['job_id']=opts.nevents_split
224	>	info = ParseInfo(samplesinfo,path)
225	>	if ( opts.samples == "" ):
226	>	for job in info:
227	>	if (job.subsample): continue #avoid multiple submissions from subsamples
228	>	submit(job.name,repDict)
229	>	else:
230	>	for sample in samplesList:
231	>	submit(sample,repDict)
232	>
233	>	#BDT optimisation
234	>	elif opts.task == 'mva_opt':
235	>	total_number_of_steps=1
236	>	setting = ''
237	>	for par in (config.get('Optimisation','parameters').split(',')):
238	>	scan_par=eval(config.get('Optimisation',par))
239	>	setting+=par+'='+str(scan_par[0])+':'
240	>	if len(scan_par) > 1 and scan_par[2] != 0:
241	>	total_number_of_steps+=scan_par[2]
242	>	setting=setting[:-1] # eliminate last column at the end of the setting string
243	>	print setting
244	>	repDict['additional']=setting
245	>	repDict['job_id']=config.get('Optimisation','training')
246	>	submit('OPT_main_set',repDict)
247	>	main_setting=setting
248	>
249	>	#Scanning all the parameters found in the training config in the Optimisation sector
250	>	for par in (config.get('Optimisation','parameters').split(',')):
251	>	scan_par=eval(config.get('Optimisation',par))
252	>	print par
253	>	if len(scan_par) > 1 and scan_par[2] != 0:
254	>	for step in range(scan_par[2]):
255	>	value = (scan_par[0])+((1+step)*(scan_par[1]-scan_par[0])/scan_par[2])
256	>	print value
257	>	setting=re.sub(par+'.*?:',par+'='+str(value)+':',main_setting)
258	>	repDict['additional']=setting
259	>	#                               repDict['job_id']=config.get('Optimisation','training')
260	>	submit('OPT_'+par+str(value),repDict)
261	>	#                               submit(config.get('Optimisation','training'),repDict)
262	>	print setting
263	>
264	>
265	>	os.system('qstat')
266	>	if (opts.philipp_love_progress_bars):
267	>	os.system('./qstat.py')

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines