ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/UserCode/VHbb/python/submitThem.py

Comparing UserCode/VHbb/python/submitThem.py (file contents):
Revision 1.24 by peller, Wed Jan 16 16:22:47 2013 UTC vs.
Revision 1.36 by nmohr, Fri Mar 22 15:19:28 2013 UTC

#	Line 1 | Line 1
1		#! /usr/bin/env python
2	–	import os,shutil,sys,pickle,subprocess,ROOT
2		from optparse import OptionParser
3	<	import getpass
4	<	from myutils import BetterConfigParser, sample, parse_info
3	>	import sys
4	>	import time
5	>	import os
6	>	import shutil
7
8		parser = OptionParser()
9		parser.add_option("-T", "--tag", dest="tag", default="",
#	Line 13 | Line 14 | parser.add_option("-M", "--mass", dest="
14		help="Mass for DC or Plots, 110...135")
15		parser.add_option("-S","--samples",dest="samples",default="",
16		help="samples you want to run on")
17	<
17	>	parser.add_option("-F", "--folderTag", dest="ftag", default="",
18	>	help="Creats a new folder structure for outputs or uses an existing one with the given name")
19	>	parser.add_option("-N", "--number-of-events", dest="nevents_split", default=100000,
20	>	help="Number of events per file when splitting.")
21	>	parser.add_option("-P", "--philipp-love-progress-bars", dest="philipp_love_progress_bars", default=False,
22	>	help="If you share the love of Philipp...")
23
24		(opts, args) = parser.parse_args(sys.argv)
25	+
26	+	import os,shutil,pickle,subprocess,ROOT,re
27	+	ROOT.gROOT.SetBatch(True)
28	+	from myutils import BetterConfigParser, Sample, ParseInfo, sample_parser
29	+	import getpass
30	+
31		if opts.tag == "":
32		print "Please provide tag to run the analysis with, example '-T 8TeV' uses config8TeV and pathConfig8TeV to run the analysis."
33		sys.exit(123)
#	Line 24 | Line 36 | if opts.task == "":
36		print "Please provide a task.\n-J prep:\tpreparation of Trees\n-J sys:\t\twrite regression and systematics\n-J eval:\tcreate MVA output\n-J plot:\tproduce Plots\n-J dc:\t\twrite workspaces and datacards"
37		sys.exit(123)
38
39	+
40	+	en = opts.tag
41	+
42		#create the list with the samples to run over
43		samplesList=opts.samples.split(",")
44	+	timestamp = time.asctime().replace(' ','_').replace(':','-')
45	+
46	+	# the list of the config is taken from the path config
47	+	pathconfig = BetterConfigParser()
48	+	pathconfig.read('%sconfig/paths'%(en))
49	+	_configs = pathconfig.get('Configuration','List').split(" ")
50	+	configs = [ '%sconfig/'%(en) + c for c in _configs  ]
51	+
52	+	if not opts.ftag == '':
53	+	tagDir = pathconfig.get('Directories','tagDir')
54	+	DirStruct={'tagDir':tagDir,'ftagdir':'%s/%s/'%(tagDir,opts.ftag),'logpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Logs'),'plotpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Plots'),'limitpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Limits'),'confpath':'%s/%s/%s/'%(tagDir,opts.ftag,'config') }
55	+
56	+	for keys in ['tagDir','ftagdir','logpath','plotpath','limitpath','confpath']:
57	+	try:
58	+	os.stat(DirStruct[keys])
59	+	except:
60	+	os.mkdir(DirStruct[keys])
61	+
62	+	pathfile = open('%sconfig/paths'%(en))
63	+	buffer = pathfile.readlines()
64	+	pathfile.close()
65	+	os.rename('%sconfig/paths'%(en),'%sconfig/paths.bkp'%(en))
66	+	pathfile = open('%sconfig/paths'%(en),'w')
67	+	for line in buffer:
68	+	if line.startswith('plotpath'):
69	+	line = 'plotpath: %s\n'%DirStruct['plotpath']
70	+	elif line.startswith('logpath'):
71	+	line = 'logpath: %s\n'%DirStruct['logpath']
72	+	elif line.startswith('limits'):
73	+	line = 'limits: %s\n'%DirStruct['limitpath']
74	+	pathfile.write(line)
75	+	pathfile.close()
76	+
77	+	#copy config files
78	+	for item in configs:
79	+	shutil.copyfile(item,'%s/%s/%s'%(tagDir,opts.ftag,item.strip(en)))
80	+
81
30	–	en = opts.tag
31	–	#configs = ['config%s'%(en),'pathConfig%s'%(en)]
32	–	configs = ['#sconfig/general'%(en),'#sconfig/paths'%(en),'#sconfig/plots'%(en),'#sconfig/training'%(en),'#sconfig/datacards'%(en),'#sconfig/cuts'%(en)]
33	–
82		print configs
83		config = BetterConfigParser()
84		config.read(configs)
85	<	btagLibrary = config.get('BTagReshaping','library')
86	<	submitDir = os.getcwd()
87	<	os.chdir(os.path.dirname(btagLibrary))
88	<	if not os.path.exists(btagLibrary):
89	<	ROOT.gROOT.LoadMacro('%s+'%btagLibrary.replace('_h.so','.h'))
90	<	shutil.copyfile(os.path.basename(btagLibrary),'/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)))
91	<	shutil.copyfile('/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)),btagLibrary)
92	<	os.chdir(submitDir)
85	>
86	>
87	>	def compile_macro(config,macro):
88	>	"""
89	>	Creates the library from a macro using CINT compiling it in scratch to avoid
90	>	problems with the linking in the working nodes.
91	>	Args:
92	>	config: configuration file where the macro path is specified
93	>	macro: macro name to be compiled
94	>	Returns:
95	>	nothing
96	>	"""
97	>	submitDir = os.getcwd()
98	>	_macro=macro+'.h'
99	>	library = config.get(macro,'library')
100	>	libDir=os.path.dirname(library)
101	>	os.chdir(libDir)
102	>	if not os.path.exists(library):
103	>	print '@INFO: Compiling ' + _macro
104	>	scratchDir='/scratch/%s/'%(getpass.getuser())
105	>	shutil.copyfile(libDir+'/'+_macro,'/scratch/%s/%s'%(getpass.getuser(),_macro))
106	>	os.chdir(scratchDir)
107	>	ROOT.gROOT.ProcessLine('.L %s+'%(scratchDir+_macro))
108	>	shutil.copyfile('/scratch/%s/%s'%(getpass.getuser(),os.path.basename(library)),library)
109	>	os.chdir(submitDir)
110	>
111	>	compile_macro(config,'BTagReshaping')
112	>	compile_macro(config,'VHbbNameSpace')
113	>
114		logPath = config.get("Directories","logpath")
115	+	logo = open('%s/data/submit.txt' %config.get('Directories','vhbbpath')).readlines()
116	+	counter = 0
117	+
118		#check if the logPath exist. If not exit
119		if( not os.path.isdir(logPath) ):
120	<	print 'ERROR: ' + logPath + ': dir not found.'
121	<	print 'ERROR: Create it before submitting '
122	<	print 'Exit'
123	<	sys.exit(-1)
120	>	print '@ERROR : ' + logPath + ': dir not found.'
121	>	print '@ERROR : Create it before submitting '
122	>	print 'Exit'
123	>	sys.exit(-1)
124	>
125
126	<	repDict = {'en':en,'logpath':logPath,'job':'','task':opts.task,'queue': 'all.q'}
126	>	repDict = {'en':en,'logpath':logPath,'job':'','task':opts.task,'queue': 'all.q','timestamp':timestamp,'additional':'','job_id':''}
127		def submit(job,repDict):
128	<	repDict['job'] = job
129	<	command = 'qsub -V -cwd -q %(queue)s -N %(job)s_%(en)s%(task)s -o %(logpath)s/%(job)s_%(en)s_%(task)s.out -e %(logpath)s/%(job)s_%(en)s_%(task)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task
130	<	print command
131	<	subprocess.call([command], shell=True)
128	>	global counter
129	>	repDict['job'] = job
130	>	nJob = counter % len(logo)
131	>	counter += 1
132	>	if opts.philipp_love_progress_bars:
133	>	repDict['name'] = '"%s"' %logo[nJob].strip()
134	>	else:
135	>	repDict['name'] = '%(job)s_%(en)s%(task)s' %repDict
136	>	command = 'qsub -V -cwd -q %(queue)s -l h_vmem=6G -N %(name)s -o %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.out -e %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task + ' ' + repDict['job_id'] + ' ' + repDict['additional']
137	>	print command
138	>	subprocess.call([command], shell=True)
139	>
140	>	if opts.task == 'train':
141	>	train_list = (config.get('MVALists','List_for_submitscript')).split(',')
142	>	print train_list
143	>	for item in train_list:
144	>	submit(item,repDict)
145	>
146
147		if opts.task == 'dc':
148		#DC_vars = config.items('Limit')
#	Line 68 | Line 155 | if opts.task == 'plot':
155		if not opts.task == 'prep':
156		path = config.get("Directories","samplepath")
157		samplesinfo = config.get("Directories","samplesinfo")
158	<	info = parse_info(samplesinfo,path)
158	>	info = ParseInfo(samplesinfo,path)
159
160		if opts.task == 'plot':
161		repDict['queue'] = 'all.q'
162		for item in Plot_vars:
163		submit(item,repDict)
164
165	<	elif opts.task == 'dc':
165	>	if opts.task == 'trainReg':
166		repDict['queue'] = 'all.q'
167	+	submit('trainReg',repDict)
168	+
169	+
170	+	elif opts.task == 'dc':
171	+	repDict['queue'] = 'short.q'
172		for item in DC_vars:
173		if 'ZH%s'%opts.mass in item:
174		submit(item,repDict)
#	Line 84 | Line 176 | elif opts.task == 'dc':
176		submit(item,repDict)
177
178		elif opts.task == 'prep':
179	<	submit('prepare',repDict)
179	>	if ( opts.samples == ""):
180	>	path = config.get("Directories","PREPin")
181	>	samplesinfo = config.get("Directories","samplesinfo")
182	>	info = ParseInfo(samplesinfo,path)
183	>	for job in info:
184	>	submit(job.name,repDict)
185
186	<	elif opts.task == 'eval' or opts.task == 'sys' or opts.task == 'syseval':
186	>	else:
187	>	for sample in samplesList:
188	>	submit(sample,repDict)
189	>	elif opts.task == 'sys' or opts.task == 'syseval':
190	>	path = config.get("Directories","SYSin")
191	>	samplesinfo = config.get("Directories","samplesinfo")
192	>	info = ParseInfo(samplesinfo,path)
193		if ( opts.samples == ""):
194		for job in info:
195	+	if (job.subsample):
196	+	continue #avoid multiple submissions form subsamples
197	+	# TO FIX FOR SPLITTED SAMPLE
198		submit(job.name,repDict)
199		else:
200		for sample in samplesList:
201		submit(sample,repDict)
202
203	<	os.system('qstat')
203	>	elif opts.task == 'eval':
204	>	path = config.get("Directories","MVAin")
205	>	samplesinfo = config.get("Directories","samplesinfo")
206	>	info = ParseInfo(samplesinfo,path)
207	>	if ( opts.samples == ""):
208	>	for job in info:
209	>	if (job.subsample):
210	>	continue #avoid multiple submissions from subsamples
211	>	if(info.checkSplittedSampleName(job.identifier)): # if multiple entries for one name  (splitted samples) use the identifier to submit
212	>	print '@INFO: Splitted samples: submit through identifier'
213	>	submit(job.identifier,repDict)
214	>	else: submit(job.name,repDict)
215	>	else:
216	>	for sample in samplesList:
217	>	submit(sample,repDict)
218	>
219	>
220	>	elif( opts.task == 'split' ):
221	>	path = config.get("Directories","SPLITin")
222	>	samplesinfo = config.get("Directories","samplesinfo")
223	>	repDict['job_id']=opts.nevents_split
224	>	info = ParseInfo(samplesinfo,path)
225	>	if ( opts.samples == "" ):
226	>	for job in info:
227	>	if (job.subsample): continue #avoid multiple submissions from subsamples
228	>	submit(job.name,repDict)
229	>	else:
230	>	for sample in samplesList:
231	>	submit(sample,repDict)
232	>
233	>	#BDT optimisation
234	>	elif opts.task == 'mva_opt':
235	>	total_number_of_steps=1
236	>	setting = ''
237	>	for par in (config.get('Optimisation','parameters').split(',')):
238	>	scan_par=eval(config.get('Optimisation',par))
239	>	setting+=par+'='+str(scan_par[0])+':'
240	>	if len(scan_par) > 1 and scan_par[2] != 0:
241	>	total_number_of_steps+=scan_par[2]
242	>	setting=setting[:-1] # eliminate last column at the end of the setting string
243	>	print setting
244	>	repDict['additional']=setting
245	>	repDict['job_id']=config.get('Optimisation','training')
246	>	submit('OPT_main_set',repDict)
247	>	main_setting=setting
248	>
249	>	#Scanning all the parameters found in the training config in the Optimisation sector
250	>	for par in (config.get('Optimisation','parameters').split(',')):
251	>	scan_par=eval(config.get('Optimisation',par))
252	>	print par
253	>	if len(scan_par) > 1 and scan_par[2] != 0:
254	>	for step in range(scan_par[2]):
255	>	value = (scan_par[0])+((1+step)*(scan_par[1]-scan_par[0])/scan_par[2])
256	>	print value
257	>	setting=re.sub(par+'.*?:',par+'='+str(value)+':',main_setting)
258	>	repDict['additional']=setting
259	>	#                               repDict['job_id']=config.get('Optimisation','training')
260	>	submit('OPT_'+par+str(value),repDict)
261	>	#                               submit(config.get('Optimisation','training'),repDict)
262	>	print setting
263	>
264	>
265	>	os.system('qstat')
266	>	if (opts.philipp_love_progress_bars):
267	>	os.system('./qstat.py')

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines