ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/UserCode/VHbb/python/submitThem.py

Comparing UserCode/VHbb/python/submitThem.py (file contents):
Revision 1.6 by nmohr, Thu Sep 20 18:20:00 2012 UTC vs.
Revision 1.29 by peller, Wed Feb 13 10:06:17 2013 UTC

#	Line 1 | Line 1
1		#! /usr/bin/env python
2	–	import os,sys,pickle,subprocess
2		from optparse import OptionParser
3	<	from BetterConfigParser import BetterConfigParser
4	<	from samplesclass import sample
3	>	import sys
4	>	import time
5	>	import os
6	>	import shutil
7
8		parser = OptionParser()
9		parser.add_option("-T", "--tag", dest="tag", default="",
10		help="Tag to run the analysis with, example '8TeV' uses config8TeV and pathConfig8TeV to run the analysis")
11	<
11	>	parser.add_option("-J", "--task", dest="task", default="",
12	>	help="Task to be done, i.e. 'dc' for Datacards, 'prep' for preparation of Trees, 'plot' to produce plots or 'eval' to write the MVA output or 'sys' to write regression and systematics (or 'syseval' for both). ")
13	>	parser.add_option("-M", "--mass", dest="mass", default="125",
14	>	help="Mass for DC or Plots, 110...135")
15	>	parser.add_option("-S","--samples",dest="samples",default="",
16	>	help="samples you want to run on")
17	>	parser.add_option("-F", "--folderTag", dest="ftag", default="",
18	>	help="Creats a new folder structure for outputs or uses an existing one with the given name")
19		(opts, args) = parser.parse_args(sys.argv)
20	+
21	+	import os,shutil,pickle,subprocess,ROOT
22	+	ROOT.gROOT.SetBatch(True)
23	+	from myutils import BetterConfigParser, Sample, ParseInfo
24	+	import getpass
25	+
26		if opts.tag == "":
27		print "Please provide tag to run the analysis with, example '-T 8TeV' uses config8TeV and pathConfig8TeV to run the analysis."
28		sys.exit(123)
29	+
30	+	if opts.task == "":
31	+	print "Please provide a task.\n-J prep:\tpreparation of Trees\n-J sys:\t\twrite regression and systematics\n-J eval:\tcreate MVA output\n-J plot:\tproduce Plots\n-J dc:\t\twrite workspaces and datacards"
32	+	sys.exit(123)
33	+
34	+
35		en = opts.tag
36	<	configs = ['config%s'%(en),'pathConfig%s'%(en)]
36	>
37	>	#create the list with the samples to run over
38	>	samplesList=opts.samples.split(",")
39	>
40	>	timestamp = time.asctime().replace(' ','_').replace(':','-')
41	>
42	>	configs = ['%sconfig/general'%(en),'%sconfig/paths'%(en),'%sconfig/plots'%(en),'%sconfig/training'%(en),'%sconfig/datacards'%(en),'%sconfig/cuts'%(en)]
43	>
44	>	pathconfig = BetterConfigParser()
45	>	pathconfig.read('%sconfig/paths'%(en))
46	>
47	>	if not opts.ftag == '':
48	>	tagDir = pathconfig.get('Directories','tagDir')
49	>	DirStruct={'tagDir':tagDir,'ftagdir':'%s/%s/'%(tagDir,opts.ftag),'logpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Logs'),'plotpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Plots'),'limitpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Limits'),'confpath':'%s/%s/%s/'%(tagDir,opts.ftag,'config') }
50	>
51	>	for keys in ['tagDir','ftagdir','logpath','plotpath','limitpath','confpath']:
52	>	try:
53	>	os.stat(DirStruct[keys])
54	>	except:
55	>	os.mkdir(DirStruct[keys])
56	>
57	>	pathfile = open('%sconfig/paths'%(en))
58	>	buffer = pathfile.readlines()
59	>	pathfile.close()
60	>	os.rename('%sconfig/paths'%(en),'%sconfig/paths.bkp'%(en))
61	>	pathfile = open('%sconfig/paths'%(en),'w')
62	>	for line in buffer:
63	>	if line.startswith('plotpath'):
64	>	line = 'plotpath: %s\n'%DirStruct['plotpath']
65	>	elif line.startswith('logpath'):
66	>	line = 'logpath: %s\n'%DirStruct['logpath']
67	>	elif line.startswith('limits'):
68	>	line = 'limits: %s\n'%DirStruct['limitpath']
69	>	pathfile.write(line)
70	>	pathfile.close()
71	>
72	>	#copy config files
73	>	for item in configs:
74	>	shutil.copyfile(item,'%s/%s/%s'%(tagDir,opts.ftag,item.strip(en)))
75	>
76	>
77		print configs
78		config = BetterConfigParser()
79		config.read(configs)
80	+
81	+	btagLibrary = config.get('BTagReshaping','library')
82	+	submitDir = os.getcwd()
83	+	os.chdir(os.path.dirname(btagLibrary))
84	+	if not os.path.exists(btagLibrary):
85	+	ROOT.gROOT.LoadMacro('%s+'%btagLibrary.replace('_h.so','.h'))
86	+	shutil.copyfile(os.path.basename(btagLibrary),'/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)))
87	+	shutil.copyfile('/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)),btagLibrary)
88	+	os.chdir(submitDir)
89		logPath = config.get("Directories","logpath")
90	<	repDict = {'en':en,'logpath':logPath,'job':''}
90	>	#check if the logPath exist. If not exit
91	>	if( not os.path.isdir(logPath) ):
92	>	print 'ERROR: ' + logPath + ': dir not found.'
93	>	print 'ERROR: Create it before submitting '
94	>	print 'Exit'
95	>	sys.exit(-1)
96	>
97	>	repDict = {'en':en,'logpath':logPath,'job':'','task':opts.task,'queue': 'all.q','timestamp':timestamp}
98		def submit(job,repDict):
99		repDict['job'] = job
100	<	command = 'qsub -V -cwd -q all.q -N %(job)s_%(en)s -o %(logpath)s/%(job)s_%(en)s.out -e %(logpath)s/%(job)s_%(en)s.err runAll.sh %(job)s %(en)s' %repDict
100	>	command = 'qsub -V -cwd -q %(queue)s -l h_vmem=6G -N %(job)s_%(en)s%(task)s -o %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.out -e %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task
101		print command
102		subprocess.call([command], shell=True)
103
104	<	path = config.get("Directories","samplepath")
105	<	infofile = open(path+'/env/samples.info','r')
106	<	info = pickle.load(infofile)
107	<	infofile.close()
104	>	if opts.task == 'train':
105	>	train_list = (config.get('MVALists','List_for_submitscript')).split(',')
106	>	print train_list
107	>	for item in train_list:
108	>	submit(item,repDict)
109	>
110	>
111	>	if opts.task == 'dc':
112	>	#DC_vars = config.items('Limit')
113	>	DC_vars= (config.get('LimitGeneral','List')).split(',')
114	>	print DC_vars
115	>
116	>	if opts.task == 'plot':
117	>	Plot_vars= (config.get('Plot_general','List')).split(',')
118	>
119	>	if not opts.task == 'prep':
120	>	path = config.get("Directories","samplepath")
121	>	samplesinfo = config.get("Directories","samplesinfo")
122	>	info = ParseInfo(samplesinfo,path)
123	>
124	>	if opts.task == 'plot':
125	>	repDict['queue'] = 'all.q'
126	>	for item in Plot_vars:
127	>	submit(item,repDict)
128	>
129	>	elif opts.task == 'dc':
130	>	repDict['queue'] = 'all.q'
131	>	for item in DC_vars:
132	>	if 'ZH%s'%opts.mass in item:
133	>	submit(item,repDict)
134	>	elif 'ZH' in item and opts.mass == 'all':
135	>	submit(item,repDict)
136	>
137	>	elif opts.task == 'prep':
138	>	submit('prepare',repDict)
139	>
140	>	elif opts.task == 'eval' or opts.task == 'sys' or opts.task == 'syseval':
141	>	if ( opts.samples == ""):
142	>	for job in info:
143	>	submit(job.name,repDict)
144	>	else:
145	>	for sample in samplesList:
146	>	submit(sample,repDict)
147
148	<	for job in info:
149	<	submit(job.name,repDict)
148	>	os.system('qstat')
149	>	os.system('./qstat.py')

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines