VHbb/python/submitThem.py

#! /usr/bin/env python
from optparse import OptionParser
import sys
import time
import os
import shutil

parser = OptionParser()
parser.add_option("-T", "--tag", dest="tag", default="",
                      help="Tag to run the analysis with, example '8TeV' uses config8TeV and pathConfig8TeV to run the analysis")
parser.add_option("-J", "--task", dest="task", default="",
                      help="Task to be done, i.e. 'dc' for Datacards, 'prep' for preparation of Trees, 'plot' to produce plots or 'eval' to write the MVA output or 'sys' to write regression and systematics (or 'syseval' for both). ")
parser.add_option("-M", "--mass", dest="mass", default="125",
                      help="Mass for DC or Plots, 110...135")
parser.add_option("-S","--samples",dest="samples",default="",
                      help="samples you want to run on")
parser.add_option("-F", "--folderTag", dest="ftag", default="",
                      help="Creats a new folder structure for outputs or uses an existing one with the given name")
parser.add_option("-N", "--number-of-events", dest="nevents_split", default=100000,
                      help="Number of events per file when splitting.")
parser.add_option("-P", "--philipp-love-progress-bars", dest="philipp_love_progress_bars", default=False,
                      help="If you share the love of Philipp...")

(opts, args) = parser.parse_args(sys.argv)

import os,shutil,pickle,subprocess,ROOT,re
ROOT.gROOT.SetBatch(True)
from myutils import BetterConfigParser, Sample, ParseInfo, sample_parser
import getpass

if opts.tag == "":
        print "Please provide tag to run the analysis with, example '-T 8TeV' uses config8TeV and pathConfig8TeV to run the analysis."
        sys.exit(123)

if opts.task == "":
    print "Please provide a task.\n-J prep:\tpreparation of Trees\n-J sys:\t\twrite regression and systematics\n-J eval:\tcreate MVA output\n-J plot:\tproduce Plots\n-J dc:\t\twrite workspaces and datacards"
    sys.exit(123)


en = opts.tag

#create the list with the samples to run over
samplesList=opts.samples.split(",")

timestamp = time.asctime().replace(' ','_').replace(':','-')

configs = ['%sconfig/general'%(en),'%sconfig/paths'%(en),'%sconfig/plots'%(en),'%sconfig/training'%(en),'%sconfig/datacards'%(en),'%sconfig/cuts'%(en)]

pathconfig = BetterConfigParser()
pathconfig.read('%sconfig/paths'%(en))

if not opts.ftag == '':
    tagDir = pathconfig.get('Directories','tagDir')
    DirStruct={'tagDir':tagDir,'ftagdir':'%s/%s/'%(tagDir,opts.ftag),'logpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Logs'),'plotpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Plots'),'limitpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Limits'),'confpath':'%s/%s/%s/'%(tagDir,opts.ftag,'config') }

    for keys in ['tagDir','ftagdir','logpath','plotpath','limitpath','confpath']:
        try:
            os.stat(DirStruct[keys])
        except:
            os.mkdir(DirStruct[keys])

    pathfile = open('%sconfig/paths'%(en))
    buffer = pathfile.readlines()
    pathfile.close()
    os.rename('%sconfig/paths'%(en),'%sconfig/paths.bkp'%(en))
    pathfile = open('%sconfig/paths'%(en),'w')
    for line in buffer:
        if line.startswith('plotpath'):
            line = 'plotpath: %s\n'%DirStruct['plotpath']
        elif line.startswith('logpath'):
            line = 'logpath: %s\n'%DirStruct['logpath']
        elif line.startswith('limits'):
            line = 'limits: %s\n'%DirStruct['limitpath']
        pathfile.write(line)
    pathfile.close()

    #copy config files
    for item in configs:
        shutil.copyfile(item,'%s/%s/%s'%(tagDir,opts.ftag,item.strip(en)))


print configs
config = BetterConfigParser()
config.read(configs)

btagLibrary = config.get('BTagReshaping','library')
submitDir = os.getcwd()
os.chdir(os.path.dirname(btagLibrary))
if not os.path.exists(btagLibrary):
    ROOT.gROOT.LoadMacro('%s+'%btagLibrary.replace('_h.so','.h')) 
shutil.copyfile(os.path.basename(btagLibrary),'/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)))
shutil.copyfile('/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)),btagLibrary)
os.chdir(submitDir)
logPath = config.get("Directories","logpath")
#check if the logPath exist. If not exit
if( not os.path.isdir(logPath) ):
        print 'ERROR: ' + logPath + ': dir not found.'
        print 'ERROR: Create it before submitting '
        print 'Exit'
        sys.exit(-1)

repDict = {'en':en,'logpath':logPath,'job':'','task':opts.task,'queue': 'all.q','timestamp':timestamp,'additional':'','job_id':''}
def submit(job,repDict):
        repDict['job'] = job
        command = 'qsub -V -cwd -q %(queue)s -l h_vmem=6G -N %(job)s_%(en)s%(task)s -o %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.out -e %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task + ' ' + repDict['job_id'] + ' ' + repDict['additional']
        print command
        subprocess.call([command], shell=True)

if opts.task == 'train':
    train_list = (config.get('MVALists','List_for_submitscript')).split(',')
    print train_list
    for item in train_list:
        submit(item,repDict)


if opts.task == 'dc':
    #DC_vars = config.items('Limit')
    DC_vars= (config.get('LimitGeneral','List')).split(',')
    print DC_vars

if opts.task == 'plot':
    Plot_vars= (config.get('Plot_general','List')).split(',')

if not opts.task == 'prep':
    path = config.get("Directories","samplepath")
    samplesinfo = config.get("Directories","samplesinfo")
    info = ParseInfo(samplesinfo,path)

if opts.task == 'plot': 
    repDict['queue'] = 'all.q'
    for item in Plot_vars:
        submit(item,repDict)


elif opts.task == 'dc':
    repDict['queue'] = 'short.q'
    for item in DC_vars:
        if 'ZH%s'%opts.mass in item:
            submit(item,repDict) 
        elif 'ZH' in item and opts.mass == 'all':
            submit(item,repDict)
            
elif opts.task == 'prep':
    if ( opts.samples == ""):
        path = config.get("Directories","PREPin")
        samplesinfo = config.get("Directories","samplesinfo")
        info = ParseInfo(samplesinfo,path)
        for job in info:
            submit(job.name,repDict)

    else:
        for sample in samplesList:
            submit(sample,repDict)
elif opts.task == 'sys' or opts.task == 'syseval':
    path = config.get("Directories","SYSin")
    samplesinfo = config.get("Directories","samplesinfo")
    info = ParseInfo(samplesinfo,path)
    if ( opts.samples == ""):
        for job in info:
            if (job.subsample): continue #avoid multiple submissions form subsamples
            # TO FIX FOR SPLITTED SAMPLE
            submit(job.name,repDict)
    else:
        for sample in samplesList:
            submit(sample,repDict)

elif opts.task == 'eval':
    path = config.get("Directories","MVAin")
    samplesinfo = config.get("Directories","samplesinfo")
    info = ParseInfo(samplesinfo,path)
    if ( opts.samples == ""):
        for job in info:
            if (job.subsample): continue #avoid multiple submissions from subsamples
            if(info.checkSplittedSampleName(job.identifier)): # if multiple entries for one name  (splitted samples) use the identifier to submit
                    print '@INFO: Splitted samples: submit through identifier'
                    submit(job.identifier,repDict)
            else: submit(job.name,repDict)
    else:
        for sample in samplesList:
            submit(sample,repDict)


elif( opts.task == 'split' ):
        path = config.get("Directories","SPLITin")
        samplesinfo = config.get("Directories","samplesinfo")
        repDict['job_id']=opts.nevents_split
        info = ParseInfo(samplesinfo,path)
        if ( opts.samples == "" ):
                for job in info:
                        if (job.subsample): continue #avoid multiple submissions from subsamples
                        submit(job.name,repDict)
        else:
                for sample in samplesList:
                        submit(sample,repDict)

#BDT optimisation
elif opts.task == 'mva_opt':
        total_number_of_steps=1
        setting = ''
        for par in (config.get('Optimisation','parameters').split(',')):
                scan_par=eval(config.get('Optimisation',par))
                setting+=par+'='+str(scan_par[0])+':'
                if len(scan_par) > 1 and scan_par[2] != 0:
                        total_number_of_steps+=scan_par[2]
        setting=setting[:-1] # eliminate last column at the end of the setting string
        print setting
        repDict['additional']=setting
        repDict['job_id']=config.get('Optimisation','training')
        submit('OPT_main_set',repDict)
        main_setting=setting

        #Scanning all the parameters found in the training config in the Optimisation sector
        for par in (config.get('Optimisation','parameters').split(',')):
                scan_par=eval(config.get('Optimisation',par))
                print par
                if len(scan_par) > 1 and scan_par[2] != 0:
                        for step in range(scan_par[2]):
                                value = (scan_par[0])+((1+step)*(scan_par[1]-scan_par[0])/scan_par[2])
                                print value
                                setting=re.sub(par+'.*?:',par+'='+str(value)+':',main_setting)
                                repDict['additional']=setting
#                               repDict['job_id']=config.get('Optimisation','training')
                                submit('OPT_'+par+str(value),repDict)
#                               submit(config.get('Optimisation','training'),repDict)
                                print setting


os.system('qstat')
if (opts.philipp_love_progress_bars):
        os.system('./qstat.py') 
Revision:	1.33
Committed:	Thu Feb 28 16:38:49 2013 UTC (12 years, 2 months ago) by nmohr
Content type:	text/x-python
Branch:	MAIN
Changes since 1.32:	+9 -1 lines
Log Message:	Prepare split jobs
#	Content
1	#! /usr/bin/env python
2	from optparse import OptionParser
3	import sys
4	import time
5	import os
6	import shutil
7
8	parser = OptionParser()
9	parser.add_option("-T", "--tag", dest="tag", default="",
10	help="Tag to run the analysis with, example '8TeV' uses config8TeV and pathConfig8TeV to run the analysis")
11	parser.add_option("-J", "--task", dest="task", default="",
12	help="Task to be done, i.e. 'dc' for Datacards, 'prep' for preparation of Trees, 'plot' to produce plots or 'eval' to write the MVA output or 'sys' to write regression and systematics (or 'syseval' for both). ")
13	parser.add_option("-M", "--mass", dest="mass", default="125",
14	help="Mass for DC or Plots, 110...135")
15	parser.add_option("-S","--samples",dest="samples",default="",
16	help="samples you want to run on")
17	parser.add_option("-F", "--folderTag", dest="ftag", default="",
18	help="Creats a new folder structure for outputs or uses an existing one with the given name")
19	parser.add_option("-N", "--number-of-events", dest="nevents_split", default=100000,
20	help="Number of events per file when splitting.")
21	parser.add_option("-P", "--philipp-love-progress-bars", dest="philipp_love_progress_bars", default=False,
22	help="If you share the love of Philipp...")
23
24	(opts, args) = parser.parse_args(sys.argv)
25
26	import os,shutil,pickle,subprocess,ROOT,re
27	ROOT.gROOT.SetBatch(True)
28	from myutils import BetterConfigParser, Sample, ParseInfo, sample_parser
29	import getpass
30
31	if opts.tag == "":
32	print "Please provide tag to run the analysis with, example '-T 8TeV' uses config8TeV and pathConfig8TeV to run the analysis."
33	sys.exit(123)
34
35	if opts.task == "":
36	print "Please provide a task.\n-J prep:\tpreparation of Trees\n-J sys:\t\twrite regression and systematics\n-J eval:\tcreate MVA output\n-J plot:\tproduce Plots\n-J dc:\t\twrite workspaces and datacards"
37	sys.exit(123)
38
39
40	en = opts.tag
41
42	#create the list with the samples to run over
43	samplesList=opts.samples.split(",")
44
45	timestamp = time.asctime().replace(' ','_').replace(':','-')
46
47	configs = ['%sconfig/general'%(en),'%sconfig/paths'%(en),'%sconfig/plots'%(en),'%sconfig/training'%(en),'%sconfig/datacards'%(en),'%sconfig/cuts'%(en)]
48
49	pathconfig = BetterConfigParser()
50	pathconfig.read('%sconfig/paths'%(en))
51
52	if not opts.ftag == '':
53	tagDir = pathconfig.get('Directories','tagDir')
54	DirStruct={'tagDir':tagDir,'ftagdir':'%s/%s/'%(tagDir,opts.ftag),'logpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Logs'),'plotpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Plots'),'limitpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Limits'),'confpath':'%s/%s/%s/'%(tagDir,opts.ftag,'config') }
55
56	for keys in ['tagDir','ftagdir','logpath','plotpath','limitpath','confpath']:
57	try:
58	os.stat(DirStruct[keys])
59	except:
60	os.mkdir(DirStruct[keys])
61
62	pathfile = open('%sconfig/paths'%(en))
63	buffer = pathfile.readlines()
64	pathfile.close()
65	os.rename('%sconfig/paths'%(en),'%sconfig/paths.bkp'%(en))
66	pathfile = open('%sconfig/paths'%(en),'w')
67	for line in buffer:
68	if line.startswith('plotpath'):
69	line = 'plotpath: %s\n'%DirStruct['plotpath']
70	elif line.startswith('logpath'):
71	line = 'logpath: %s\n'%DirStruct['logpath']
72	elif line.startswith('limits'):
73	line = 'limits: %s\n'%DirStruct['limitpath']
74	pathfile.write(line)
75	pathfile.close()
76
77	#copy config files
78	for item in configs:
79	shutil.copyfile(item,'%s/%s/%s'%(tagDir,opts.ftag,item.strip(en)))
80
81
82	print configs
83	config = BetterConfigParser()
84	config.read(configs)
85
86	btagLibrary = config.get('BTagReshaping','library')
87	submitDir = os.getcwd()
88	os.chdir(os.path.dirname(btagLibrary))
89	if not os.path.exists(btagLibrary):
90	ROOT.gROOT.LoadMacro('%s+'%btagLibrary.replace('_h.so','.h'))
91	shutil.copyfile(os.path.basename(btagLibrary),'/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)))
92	shutil.copyfile('/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)),btagLibrary)
93	os.chdir(submitDir)
94	logPath = config.get("Directories","logpath")
95	#check if the logPath exist. If not exit
96	if( not os.path.isdir(logPath) ):
97	print 'ERROR: ' + logPath + ': dir not found.'
98	print 'ERROR: Create it before submitting '
99	print 'Exit'
100	sys.exit(-1)
101
102	repDict = {'en':en,'logpath':logPath,'job':'','task':opts.task,'queue': 'all.q','timestamp':timestamp,'additional':'','job_id':''}
103	def submit(job,repDict):
104	repDict['job'] = job
105	command = 'qsub -V -cwd -q %(queue)s -l h_vmem=6G -N %(job)s_%(en)s%(task)s -o %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.out -e %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task + ' ' + repDict['job_id'] + ' ' + repDict['additional']
106	print command
107	subprocess.call([command], shell=True)
108
109	if opts.task == 'train':
110	train_list = (config.get('MVALists','List_for_submitscript')).split(',')
111	print train_list
112	for item in train_list:
113	submit(item,repDict)
114
115
116	if opts.task == 'dc':
117	#DC_vars = config.items('Limit')
118	DC_vars= (config.get('LimitGeneral','List')).split(',')
119	print DC_vars
120
121	if opts.task == 'plot':
122	Plot_vars= (config.get('Plot_general','List')).split(',')
123
124	if not opts.task == 'prep':
125	path = config.get("Directories","samplepath")
126	samplesinfo = config.get("Directories","samplesinfo")
127	info = ParseInfo(samplesinfo,path)
128
129	if opts.task == 'plot':
130	repDict['queue'] = 'all.q'
131	for item in Plot_vars:
132	submit(item,repDict)
133
134
135	elif opts.task == 'dc':
136	repDict['queue'] = 'short.q'
137	for item in DC_vars:
138	if 'ZH%s'%opts.mass in item:
139	submit(item,repDict)
140	elif 'ZH' in item and opts.mass == 'all':
141	submit(item,repDict)
142
143	elif opts.task == 'prep':
144	if ( opts.samples == ""):
145	path = config.get("Directories","PREPin")
146	samplesinfo = config.get("Directories","samplesinfo")
147	info = ParseInfo(samplesinfo,path)
148	for job in info:
149	submit(job.name,repDict)
150
151	else:
152	for sample in samplesList:
153	submit(sample,repDict)
154	elif opts.task == 'sys' or opts.task == 'syseval':
155	path = config.get("Directories","SYSin")
156	samplesinfo = config.get("Directories","samplesinfo")
157	info = ParseInfo(samplesinfo,path)
158	if ( opts.samples == ""):
159	for job in info:
160	if (job.subsample): continue #avoid multiple submissions form subsamples
161	# TO FIX FOR SPLITTED SAMPLE
162	submit(job.name,repDict)
163	else:
164	for sample in samplesList:
165	submit(sample,repDict)
166
167	elif opts.task == 'eval':
168	path = config.get("Directories","MVAin")
169	samplesinfo = config.get("Directories","samplesinfo")
170	info = ParseInfo(samplesinfo,path)
171	if ( opts.samples == ""):
172	for job in info:
173	if (job.subsample): continue #avoid multiple submissions from subsamples
174	if(info.checkSplittedSampleName(job.identifier)): # if multiple entries for one name (splitted samples) use the identifier to submit
175	print '@INFO: Splitted samples: submit through identifier'
176	submit(job.identifier,repDict)
177	else: submit(job.name,repDict)
178	else:
179	for sample in samplesList:
180	submit(sample,repDict)
181
182
183	elif( opts.task == 'split' ):
184	path = config.get("Directories","SPLITin")
185	samplesinfo = config.get("Directories","samplesinfo")
186	repDict['job_id']=opts.nevents_split
187	info = ParseInfo(samplesinfo,path)
188	if ( opts.samples == "" ):
189	for job in info:
190	if (job.subsample): continue #avoid multiple submissions from subsamples
191	submit(job.name,repDict)
192	else:
193	for sample in samplesList:
194	submit(sample,repDict)
195
196	#BDT optimisation
197	elif opts.task == 'mva_opt':
198	total_number_of_steps=1
199	setting = ''
200	for par in (config.get('Optimisation','parameters').split(',')):
201	scan_par=eval(config.get('Optimisation',par))
202	setting+=par+'='+str(scan_par[0])+':'
203	if len(scan_par) > 1 and scan_par[2] != 0:
204	total_number_of_steps+=scan_par[2]
205	setting=setting[:-1] # eliminate last column at the end of the setting string
206	print setting
207	repDict['additional']=setting
208	repDict['job_id']=config.get('Optimisation','training')
209	submit('OPT_main_set',repDict)
210	main_setting=setting
211
212	#Scanning all the parameters found in the training config in the Optimisation sector
213	for par in (config.get('Optimisation','parameters').split(',')):
214	scan_par=eval(config.get('Optimisation',par))
215	print par
216	if len(scan_par) > 1 and scan_par[2] != 0:
217	for step in range(scan_par[2]):
218	value = (scan_par[0])+((1+step)*(scan_par[1]-scan_par[0])/scan_par[2])
219	print value
220	setting=re.sub(par+'.*?:',par+'='+str(value)+':',main_setting)
221	repDict['additional']=setting
222	# repDict['job_id']=config.get('Optimisation','training')
223	submit('OPT_'+par+str(value),repDict)
224	# submit(config.get('Optimisation','training'),repDict)
225	print setting
226
227
228	os.system('qstat')
229	if (opts.philipp_love_progress_bars):
230	os.system('./qstat.py')