ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/UserCode/VHbb/python/submitThem.py

Comparing UserCode/VHbb/python/submitThem.py (file contents):
Revision 1.27 by nmohr, Fri Feb 1 10:47:38 2013 UTC vs.
Revision 1.35 by bortigno, Thu Mar 21 14:18:17 2013 UTC

#	Line 1 | Line 1
1		#! /usr/bin/env python
2		from optparse import OptionParser
3		import sys
4	+	import time
5	+	import os
6	+	import shutil
7
8		parser = OptionParser()
9		parser.add_option("-T", "--tag", dest="tag", default="",
#	Line 11 | Line 14 | parser.add_option("-M", "--mass", dest="
14		help="Mass for DC or Plots, 110...135")
15		parser.add_option("-S","--samples",dest="samples",default="",
16		help="samples you want to run on")
17	+	parser.add_option("-F", "--folderTag", dest="ftag", default="",
18	+	help="Creats a new folder structure for outputs or uses an existing one with the given name")
19	+	parser.add_option("-N", "--number-of-events", dest="nevents_split", default=100000,
20	+	help="Number of events per file when splitting.")
21	+	parser.add_option("-P", "--philipp-love-progress-bars", dest="philipp_love_progress_bars", default=False,
22	+	help="If you share the love of Philipp...")
23
24		(opts, args) = parser.parse_args(sys.argv)
25
26	<	import os,shutil,pickle,subprocess,ROOT
26	>	import os,shutil,pickle,subprocess,ROOT,re
27		ROOT.gROOT.SetBatch(True)
28	<	from myutils import BetterConfigParser, Sample, ParseInfo
28	>	from myutils import BetterConfigParser, Sample, ParseInfo, sample_parser
29		import getpass
30
31		if opts.tag == "":
#	Line 27 | Line 36 | if opts.task == "":
36		print "Please provide a task.\n-J prep:\tpreparation of Trees\n-J sys:\t\twrite regression and systematics\n-J eval:\tcreate MVA output\n-J plot:\tproduce Plots\n-J dc:\t\twrite workspaces and datacards"
37		sys.exit(123)
38
39	+
40	+	en = opts.tag
41	+
42		#create the list with the samples to run over
43		samplesList=opts.samples.split(",")
44	+	timestamp = time.asctime().replace(' ','_').replace(':','-')
45	+
46	+	# the list of the config is taken from the path config
47	+	pathconfig = BetterConfigParser()
48	+	pathconfig.read('%sconfig/paths'%(en))
49	+	_configs = pathconfig.get('Configuration','List').split(" ")
50	+	configs = [ '%sconfig/'%(en) + c for c in _configs  ]
51	+
52	+	if not opts.ftag == '':
53	+	tagDir = pathconfig.get('Directories','tagDir')
54	+	DirStruct={'tagDir':tagDir,'ftagdir':'%s/%s/'%(tagDir,opts.ftag),'logpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Logs'),'plotpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Plots'),'limitpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Limits'),'confpath':'%s/%s/%s/'%(tagDir,opts.ftag,'config') }
55	+
56	+	for keys in ['tagDir','ftagdir','logpath','plotpath','limitpath','confpath']:
57	+	try:
58	+	os.stat(DirStruct[keys])
59	+	except:
60	+	os.mkdir(DirStruct[keys])
61	+
62	+	pathfile = open('%sconfig/paths'%(en))
63	+	buffer = pathfile.readlines()
64	+	pathfile.close()
65	+	os.rename('%sconfig/paths'%(en),'%sconfig/paths.bkp'%(en))
66	+	pathfile = open('%sconfig/paths'%(en),'w')
67	+	for line in buffer:
68	+	if line.startswith('plotpath'):
69	+	line = 'plotpath: %s\n'%DirStruct['plotpath']
70	+	elif line.startswith('logpath'):
71	+	line = 'logpath: %s\n'%DirStruct['logpath']
72	+	elif line.startswith('limits'):
73	+	line = 'limits: %s\n'%DirStruct['limitpath']
74	+	pathfile.write(line)
75	+	pathfile.close()
76	+
77	+	#copy config files
78	+	for item in configs:
79	+	shutil.copyfile(item,'%s/%s/%s'%(tagDir,opts.ftag,item.strip(en)))
80	+
81
33	–	en = opts.tag
34	–	configs = ['%sconfig/general'%(en),'%sconfig/paths'%(en),'%sconfig/plots'%(en),'%sconfig/training'%(en),'%sconfig/datacards'%(en),'%sconfig/cuts'%(en)]
35	–
82		print configs
83		config = BetterConfigParser()
84		config.read(configs)
85	<	btagLibrary = config.get('BTagReshaping','library')
86	<	submitDir = os.getcwd()
87	<	os.chdir(os.path.dirname(btagLibrary))
88	<	if not os.path.exists(btagLibrary):
89	<	ROOT.gROOT.LoadMacro('%s+'%btagLibrary.replace('_h.so','.h'))
90	<	shutil.copyfile(os.path.basename(btagLibrary),'/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)))
91	<	shutil.copyfile('/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)),btagLibrary)
92	<	os.chdir(submitDir)
85	>
86	>
87	>	def compile_macro(config,macro):
88	>	"""
89	>	Creates the library from a macro using CINT compiling it in scratch to avoid
90	>	problems with the linking in the working nodes.
91	>	Args:
92	>	config: configuration file where the macro path is specified
93	>	macro: macro name to be compiled
94	>	Returns:
95	>	nothing
96	>	"""
97	>	submitDir = os.getcwd()
98	>	_macro=macro+'.h'
99	>	library = config.get(macro,'library')
100	>	libDir=os.path.dirname(library)
101	>	os.chdir(libDir)
102	>	if not os.path.exists(library):
103	>	print '@INFO: Compiling ' + _macro
104	>	scratchDir='/scratch/%s/'%(getpass.getuser())
105	>	shutil.copyfile(libDir+'/'+_macro,'/scratch/%s/%s'%(getpass.getuser(),_macro))
106	>	os.chdir(scratchDir)
107	>	ROOT.gROOT.ProcessLine('.L %s+'%(scratchDir+_macro))
108	>	shutil.copyfile('/scratch/%s/%s'%(getpass.getuser(),os.path.basename(library)),library)
109	>	os.chdir(submitDir)
110	>
111	>	compile_macro(config,'BTagReshaping')
112	>	compile_macro(config,'VHbbNameSpace')
113	>
114		logPath = config.get("Directories","logpath")
115	+	logo = open('%s/data/submit.txt' %config.get('Directories','vhbbpath')).readlines()
116	+	counter = 0
117	+
118		#check if the logPath exist. If not exit
119		if( not os.path.isdir(logPath) ):
120	<	print 'ERROR: ' + logPath + ': dir not found.'
121	<	print 'ERROR: Create it before submitting '
122	<	print 'Exit'
123	<	sys.exit(-1)
120	>	print '@ERROR : ' + logPath + ': dir not found.'
121	>	print '@ERROR : Create it before submitting '
122	>	print 'Exit'
123	>	sys.exit(-1)
124	>
125
126	<	repDict = {'en':en,'logpath':logPath,'job':'','task':opts.task,'queue': 'all.q'}
126	>	repDict = {'en':en,'logpath':logPath,'job':'','task':opts.task,'queue': 'all.q','timestamp':timestamp,'additional':'','job_id':''}
127		def submit(job,repDict):
128	<	repDict['job'] = job
129	<	command = 'qsub -V -cwd -q %(queue)s -l h_vmem=6G -N %(job)s_%(en)s%(task)s -o %(logpath)s/%(job)s_%(en)s_%(task)s.out -e %(logpath)s/%(job)s_%(en)s_%(task)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task
130	<	print command
131	<	subprocess.call([command], shell=True)
128	>	global counter
129	>	repDict['job'] = job
130	>	nJob = counter % len(logo)
131	>	counter += 1
132	>	if opts.philipp_love_progress_bars:
133	>	repDict['name'] = '"%s"' %logo[nJob].strip()
134	>	else:
135	>	repDict['name'] = '%(job)s_%(en)s%(task)s' %repDict
136	>	command = 'qsub -V -cwd -q %(queue)s -l h_vmem=6G -N %(name)s -o %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.out -e %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task + ' ' + repDict['job_id'] + ' ' + repDict['additional']
137	>	print command
138	>	subprocess.call([command], shell=True)
139	>
140	>	if opts.task == 'train':
141	>	train_list = (config.get('MVALists','List_for_submitscript')).split(',')
142	>	print train_list
143	>	for item in train_list:
144	>	submit(item,repDict)
145	>
146
147		if opts.task == 'dc':
148		#DC_vars = config.items('Limit')
#	Line 77 | Line 162 | if opts.task == 'plot':
162		for item in Plot_vars:
163		submit(item,repDict)
164
165	+
166		elif opts.task == 'dc':
167	<	repDict['queue'] = 'all.q'
167	>	repDict['queue'] = 'short.q'
168		for item in DC_vars:
169		if 'ZH%s'%opts.mass in item:
170		submit(item,repDict)
#	Line 86 | Line 172 | elif opts.task == 'dc':
172		submit(item,repDict)
173
174		elif opts.task == 'prep':
175	<	submit('prepare',repDict)
175	>	if ( opts.samples == ""):
176	>	path = config.get("Directories","PREPin")
177	>	samplesinfo = config.get("Directories","samplesinfo")
178	>	info = ParseInfo(samplesinfo,path)
179	>	for job in info:
180	>	submit(job.name,repDict)
181
182	<	elif opts.task == 'eval' or opts.task == 'sys' or opts.task == 'syseval':
182	>	else:
183	>	for sample in samplesList:
184	>	submit(sample,repDict)
185	>	elif opts.task == 'sys' or opts.task == 'syseval':
186	>	path = config.get("Directories","SYSin")
187	>	samplesinfo = config.get("Directories","samplesinfo")
188	>	info = ParseInfo(samplesinfo,path)
189		if ( opts.samples == ""):
190		for job in info:
191	+	if (job.subsample):
192	+	continue #avoid multiple submissions form subsamples
193	+	# TO FIX FOR SPLITTED SAMPLE
194		submit(job.name,repDict)
195		else:
196		for sample in samplesList:
197		submit(sample,repDict)
198
199	<	os.system('qstat')
199	>	elif opts.task == 'eval':
200	>	path = config.get("Directories","MVAin")
201	>	samplesinfo = config.get("Directories","samplesinfo")
202	>	info = ParseInfo(samplesinfo,path)
203	>	if ( opts.samples == ""):
204	>	for job in info:
205	>	if (job.subsample):
206	>	continue #avoid multiple submissions from subsamples
207	>	if(info.checkSplittedSampleName(job.identifier)): # if multiple entries for one name  (splitted samples) use the identifier to submit
208	>	print '@INFO: Splitted samples: submit through identifier'
209	>	submit(job.identifier,repDict)
210	>	else: submit(job.name,repDict)
211	>	else:
212	>	for sample in samplesList:
213	>	submit(sample,repDict)
214	>
215	>
216	>	elif( opts.task == 'split' ):
217	>	path = config.get("Directories","SPLITin")
218	>	samplesinfo = config.get("Directories","samplesinfo")
219	>	repDict['job_id']=opts.nevents_split
220	>	info = ParseInfo(samplesinfo,path)
221	>	if ( opts.samples == "" ):
222	>	for job in info:
223	>	if (job.subsample): continue #avoid multiple submissions from subsamples
224	>	submit(job.name,repDict)
225	>	else:
226	>	for sample in samplesList:
227	>	submit(sample,repDict)
228	>
229	>	#BDT optimisation
230	>	elif opts.task == 'mva_opt':
231	>	total_number_of_steps=1
232	>	setting = ''
233	>	for par in (config.get('Optimisation','parameters').split(',')):
234	>	scan_par=eval(config.get('Optimisation',par))
235	>	setting+=par+'='+str(scan_par[0])+':'
236	>	if len(scan_par) > 1 and scan_par[2] != 0:
237	>	total_number_of_steps+=scan_par[2]
238	>	setting=setting[:-1] # eliminate last column at the end of the setting string
239	>	print setting
240	>	repDict['additional']=setting
241	>	repDict['job_id']=config.get('Optimisation','training')
242	>	submit('OPT_main_set',repDict)
243	>	main_setting=setting
244	>
245	>	#Scanning all the parameters found in the training config in the Optimisation sector
246	>	for par in (config.get('Optimisation','parameters').split(',')):
247	>	scan_par=eval(config.get('Optimisation',par))
248	>	print par
249	>	if len(scan_par) > 1 and scan_par[2] != 0:
250	>	for step in range(scan_par[2]):
251	>	value = (scan_par[0])+((1+step)*(scan_par[1]-scan_par[0])/scan_par[2])
252	>	print value
253	>	setting=re.sub(par+'.*?:',par+'='+str(value)+':',main_setting)
254	>	repDict['additional']=setting
255	>	#                               repDict['job_id']=config.get('Optimisation','training')
256	>	submit('OPT_'+par+str(value),repDict)
257	>	#                               submit(config.get('Optimisation','training'),repDict)
258	>	print setting
259	>
260	>
261	>	os.system('qstat')
262	>	if (opts.philipp_love_progress_bars):
263	>	os.system('./qstat.py')

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines