VHbb/python/submitThem.py

#! /usr/bin/env python
from optparse import OptionParser
import sys
import time
import os
import shutil

parser = OptionParser()
parser.add_option("-T", "--tag", dest="tag", default="",
                      help="Tag to run the analysis with, example '8TeV' uses config8TeV and pathConfig8TeV to run the analysis")
parser.add_option("-J", "--task", dest="task", default="",
                      help="Task to be done, i.e. 'dc' for Datacards, 'prep' for preparation of Trees, 'plot' to produce plots or 'eval' to write the MVA output or 'sys' to write regression and systematics (or 'syseval' for both). ")
parser.add_option("-M", "--mass", dest="mass", default="125",
                      help="Mass for DC or Plots, 110...135")
parser.add_option("-S","--samples",dest="samples",default="",
                      help="samples you want to run on")
parser.add_option("-F", "--folderTag", dest="ftag", default="",
                      help="Creats a new folder structure for outputs or uses an existing one with the given name")
parser.add_option("-N", "--number-of-events", dest="nevents_split", default=100000,
                      help="Number of events per file when splitting.")
parser.add_option("-P", "--philipp-love-progress-bars", dest="philipp_love_progress_bars", default=False,
                      help="If you share the love of Philipp...")

(opts, args) = parser.parse_args(sys.argv)

import os,shutil,pickle,subprocess,ROOT,re
ROOT.gROOT.SetBatch(True)
from myutils import BetterConfigParser, Sample, ParseInfo, sample_parser
import getpass

if opts.tag == "":
        print "Please provide tag to run the analysis with, example '-T 8TeV' uses config8TeV and pathConfig8TeV to run the analysis."
        sys.exit(123)

if opts.task == "":
    print "Please provide a task.\n-J prep:\tpreparation of Trees\n-J sys:\t\twrite regression and systematics\n-J eval:\tcreate MVA output\n-J plot:\tproduce Plots\n-J dc:\t\twrite workspaces and datacards"
    sys.exit(123)


en = opts.tag

#create the list with the samples to run over
samplesList=opts.samples.split(",")

timestamp = time.asctime().replace(' ','_').replace(':','-')

configs = ['%sconfig/general'%(en),'%sconfig/paths'%(en),'%sconfig/plots'%(en),'%sconfig/training'%(en),'%sconfig/datacards'%(en),'%sconfig/cuts'%(en)]

pathconfig = BetterConfigParser()
pathconfig.read('%sconfig/paths'%(en))

if not opts.ftag == '':
    tagDir = pathconfig.get('Directories','tagDir')
    DirStruct={'tagDir':tagDir,'ftagdir':'%s/%s/'%(tagDir,opts.ftag),'logpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Logs'),'plotpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Plots'),'limitpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Limits'),'confpath':'%s/%s/%s/'%(tagDir,opts.ftag,'config') }

    for keys in ['tagDir','ftagdir','logpath','plotpath','limitpath','confpath']:
        try:
            os.stat(DirStruct[keys])
        except:
            os.mkdir(DirStruct[keys])

    pathfile = open('%sconfig/paths'%(en))
    buffer = pathfile.readlines()
    pathfile.close()
    os.rename('%sconfig/paths'%(en),'%sconfig/paths.bkp'%(en))
    pathfile = open('%sconfig/paths'%(en),'w')
    for line in buffer:
        if line.startswith('plotpath'):
            line = 'plotpath: %s\n'%DirStruct['plotpath']
        elif line.startswith('logpath'):
            line = 'logpath: %s\n'%DirStruct['logpath']
        elif line.startswith('limits'):
            line = 'limits: %s\n'%DirStruct['limitpath']
        pathfile.write(line)
    pathfile.close()

    #copy config files
    for item in configs:
        shutil.copyfile(item,'%s/%s/%s'%(tagDir,opts.ftag,item.strip(en)))


print configs
config = BetterConfigParser()
config.read(configs)

btagLibrary = config.get('BTagReshaping','library')
submitDir = os.getcwd()
os.chdir(os.path.dirname(btagLibrary))
if not os.path.exists(btagLibrary):
    ROOT.gROOT.LoadMacro('%s+'%btagLibrary.replace('_h.so','.h')) 
shutil.copyfile(os.path.basename(btagLibrary),'/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)))
shutil.copyfile('/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)),btagLibrary)
os.chdir(submitDir)
logPath = config.get("Directories","logpath")
#check if the logPath exist. If not exit
if( not os.path.isdir(logPath) ):
        print 'ERROR: ' + logPath + ': dir not found.'
        print 'ERROR: Create it before submitting '
        print 'Exit'
        sys.exit(-1)

repDict = {'en':en,'logpath':logPath,'job':'','task':opts.task,'queue': 'all.q','timestamp':timestamp,'additional':'','job_id':''}
def submit(job,repDict):
        repDict['job'] = job
        command = 'qsub -V -cwd -q %(queue)s -l h_vmem=6G -N %(job)s_%(en)s%(task)s -o %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.out -e %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task + ' ' + repDict['job_id'] + ' ' + repDict['additional']
        print command
        subprocess.call([command], shell=True)

if opts.task == 'train':
    train_list = (config.get('MVALists','List_for_submitscript')).split(',')
    print train_list
    for item in train_list:
        submit(item,repDict)


if opts.task == 'dc':
    #DC_vars = config.items('Limit')
    DC_vars= (config.get('LimitGeneral','List')).split(',')
    print DC_vars

if opts.task == 'plot':
    Plot_vars= (config.get('Plot_general','List')).split(',')

if not opts.task == 'prep':
    path = config.get("Directories","samplepath")
    samplesinfo = config.get("Directories","samplesinfo")
    info = ParseInfo(samplesinfo,path)

if opts.task == 'plot': 
    repDict['queue'] = 'all.q'
    for item in Plot_vars:
        submit(item,repDict)


elif opts.task == 'dc':
    repDict['queue'] = 'short.q'
    for item in DC_vars:
        if 'ZH%s'%opts.mass in item:
            submit(item,repDict) 
        elif 'ZH' in item and opts.mass == 'all':
            submit(item,repDict)
            
elif opts.task == 'prep':
    submit('prepare',repDict)

elif opts.task == 'sys' or opts.task == 'syseval':
    path = config.get("Directories","SYSin")
    samplesinfo = config.get("Directories","samplesinfo")
    info = ParseInfo(samplesinfo,path)
    if ( opts.samples == ""):
        for job in info:
            if (job.subsample): continue #avoid multiple submissions form subsamples
            # TO FIX FOR SPLITTED SAMPLE
            submit(job.name,repDict)
    else:
        for sample in samplesList:
            submit(sample,repDict)

elif opts.task == 'eval':
    path = config.get("Directories","MVAin")
    samplesinfo = config.get("Directories","samplesinfo")
    info = ParseInfo(samplesinfo,path)
    if ( opts.samples == ""):
        for job in info:
            if (job.subsample): continue #avoid multiple submissions from subsamples
            if(info.checkSplittedSampleName(job.identifier)): # if multiple entries for one name  (splitted samples) use the identifier to submit
                    print '@INFO: Splitted samples: submit through identifier'
                    submit(job.identifier,repDict)
            else: submit(job.name,repDict)
    else:
        for sample in samplesList:
            submit(sample,repDict)


elif( opts.task == 'split' ):
        path = config.get("Directories","SPLITin")
        samplesinfo = config.get("Directories","samplesinfo")
        repDict['job_id']=opts.nevents_split
        info = ParseInfo(samplesinfo,path)
        if ( opts.samples == "" ):
                for job in info:
                        if (job.subsample): continue #avoid multiple submissions from subsamples
                        submit(job.name,repDict)
        else:
                for sample in samplesList:
                        submit(sample,repDict)

#BDT optimisation
elif opts.task == 'mva_opt':
        total_number_of_steps=1
        setting = ''
        for par in (config.get('Optimisation','parameters').split(',')):
                scan_par=eval(config.get('Optimisation',par))
                setting+=par+'='+str(scan_par[0])+':'
                if len(scan_par) > 1 and scan_par[2] != 0:
                        total_number_of_steps+=scan_par[2]
        setting=setting[:-1] # eliminate last column at the end of the setting string
        print setting
        repDict['additional']=setting
        repDict['job_id']=config.get('Optimisation','training')
        submit('OPT_main_set',repDict)
        main_setting=setting

        #Scanning all the parameters found in the training config in the Optimisation sector
        for par in (config.get('Optimisation','parameters').split(',')):
                scan_par=eval(config.get('Optimisation',par))
                print par
                if len(scan_par) > 1 and scan_par[2] != 0:
                        for step in range(scan_par[2]):
                                value = (scan_par[0])+((1+step)*(scan_par[1]-scan_par[0])/scan_par[2])
                                print value
                                setting=re.sub(par+'.*?:',par+'='+str(value)+':',main_setting)
                                repDict['additional']=setting
#                               repDict['job_id']=config.get('Optimisation','training')
                                submit('OPT_'+par+str(value),repDict)
#                               submit(config.get('Optimisation','training'),repDict)
                                print setting


os.system('qstat')
if (opts.philipp_love_progress_bars):
        os.system('./qstat.py') 
Revision:	1.32
Committed:	Tue Feb 19 17:23:59 2013 UTC (12 years, 2 months ago) by bortigno
Content type:	text/x-python
Branch:	MAIN
Changes since 1.31:	+4 -4 lines
Log Message:	@MINOR FIXES
#	User	Rev	Content
1	nmohr	1.1	#! /usr/bin/env python
2	nmohr	1.6	from optparse import OptionParser
3	bortigno	1.26	import sys
4	peller	1.28	import time
5			import os
6			import shutil
7	nmohr	1.7
8	nmohr	1.6	parser = OptionParser()
9			parser.add_option("-T", "--tag", dest="tag", default="",
10			help="Tag to run the analysis with, example '8TeV' uses config8TeV and pathConfig8TeV to run the analysis")
11	peller	1.9	parser.add_option("-J", "--task", dest="task", default="",
12	nmohr	1.19	help="Task to be done, i.e. 'dc' for Datacards, 'prep' for preparation of Trees, 'plot' to produce plots or 'eval' to write the MVA output or 'sys' to write regression and systematics (or 'syseval' for both). ")
13	peller	1.9	parser.add_option("-M", "--mass", dest="mass", default="125",
14	bortigno	1.10	help="Mass for DC or Plots, 110...135")
15			parser.add_option("-S","--samples",dest="samples",default="",
16			help="samples you want to run on")
17	peller	1.28	parser.add_option("-F", "--folderTag", dest="ftag", default="",
18			help="Creats a new folder structure for outputs or uses an existing one with the given name")
19	bortigno	1.30	parser.add_option("-N", "--number-of-events", dest="nevents_split", default=100000,
20			help="Number of events per file when splitting.")
21			parser.add_option("-P", "--philipp-love-progress-bars", dest="philipp_love_progress_bars", default=False,
22			help="If you share the love of Philipp...")
23
24	bortigno	1.26	(opts, args) = parser.parse_args(sys.argv)
25
26	bortigno	1.30	import os,shutil,pickle,subprocess,ROOT,re
27	nmohr	1.27	ROOT.gROOT.SetBatch(True)
28	bortigno	1.31	from myutils import BetterConfigParser, Sample, ParseInfo, sample_parser
29	bortigno	1.26	import getpass
30	nmohr	1.6
31			if opts.tag == "":
32			print "Please provide tag to run the analysis with, example '-T 8TeV' uses config8TeV and pathConfig8TeV to run the analysis."
33			sys.exit(123)
34	peller	1.9
35			if opts.task == "":
36			print "Please provide a task.\n-J prep:\tpreparation of Trees\n-J sys:\t\twrite regression and systematics\n-J eval:\tcreate MVA output\n-J plot:\tproduce Plots\n-J dc:\t\twrite workspaces and datacards"
37			sys.exit(123)
38
39	peller	1.28
40			en = opts.tag
41
42	bortigno	1.11	#create the list with the samples to run over
43			samplesList=opts.samples.split(",")
44	bortigno	1.10
45	peller	1.28	timestamp = time.asctime().replace(' ','_').replace(':','-')
46
47	nmohr	1.25	configs = ['%sconfig/general'%(en),'%sconfig/paths'%(en),'%sconfig/plots'%(en),'%sconfig/training'%(en),'%sconfig/datacards'%(en),'%sconfig/cuts'%(en)]
48	peller	1.28
49			pathconfig = BetterConfigParser()
50			pathconfig.read('%sconfig/paths'%(en))
51
52			if not opts.ftag == '':
53			tagDir = pathconfig.get('Directories','tagDir')
54			DirStruct={'tagDir':tagDir,'ftagdir':'%s/%s/'%(tagDir,opts.ftag),'logpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Logs'),'plotpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Plots'),'limitpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Limits'),'confpath':'%s/%s/%s/'%(tagDir,opts.ftag,'config') }
55
56			for keys in ['tagDir','ftagdir','logpath','plotpath','limitpath','confpath']:
57			try:
58			os.stat(DirStruct[keys])
59			except:
60			os.mkdir(DirStruct[keys])
61
62			pathfile = open('%sconfig/paths'%(en))
63			buffer = pathfile.readlines()
64			pathfile.close()
65			os.rename('%sconfig/paths'%(en),'%sconfig/paths.bkp'%(en))
66			pathfile = open('%sconfig/paths'%(en),'w')
67			for line in buffer:
68			if line.startswith('plotpath'):
69			line = 'plotpath: %s\n'%DirStruct['plotpath']
70			elif line.startswith('logpath'):
71			line = 'logpath: %s\n'%DirStruct['logpath']
72			elif line.startswith('limits'):
73			line = 'limits: %s\n'%DirStruct['limitpath']
74			pathfile.write(line)
75			pathfile.close()
76
77			#copy config files
78			for item in configs:
79			shutil.copyfile(item,'%s/%s/%s'%(tagDir,opts.ftag,item.strip(en)))
80
81
82	nmohr	1.6	print configs
83			config = BetterConfigParser()
84			config.read(configs)
85	peller	1.28
86	nmohr	1.7	btagLibrary = config.get('BTagReshaping','library')
87			submitDir = os.getcwd()
88			os.chdir(os.path.dirname(btagLibrary))
89			if not os.path.exists(btagLibrary):
90			ROOT.gROOT.LoadMacro('%s+'%btagLibrary.replace('_h.so','.h'))
91			shutil.copyfile(os.path.basename(btagLibrary),'/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)))
92			shutil.copyfile('/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)),btagLibrary)
93			os.chdir(submitDir)
94	nmohr	1.6	logPath = config.get("Directories","logpath")
95	bortigno	1.14	#check if the logPath exist. If not exit
96			if( not os.path.isdir(logPath) ):
97			print 'ERROR: ' + logPath + ': dir not found.'
98			print 'ERROR: Create it before submitting '
99			print 'Exit'
100			sys.exit(-1)
101
102	bortigno	1.30	repDict = {'en':en,'logpath':logPath,'job':'','task':opts.task,'queue': 'all.q','timestamp':timestamp,'additional':'','job_id':''}
103	nmohr	1.6	def submit(job,repDict):
104			repDict['job'] = job
105	bortigno	1.30	command = 'qsub -V -cwd -q %(queue)s -l h_vmem=6G -N %(job)s_%(en)s%(task)s -o %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.out -e %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task + ' ' + repDict['job_id'] + ' ' + repDict['additional']
106	nmohr	1.1	print command
107			subprocess.call([command], shell=True)
108
109	peller	1.29	if opts.task == 'train':
110			train_list = (config.get('MVALists','List_for_submitscript')).split(',')
111			print train_list
112			for item in train_list:
113			submit(item,repDict)
114
115
116	peller	1.9	if opts.task == 'dc':
117	peller	1.18	#DC_vars = config.items('Limit')
118			DC_vars= (config.get('LimitGeneral','List')).split(',')
119			print DC_vars
120
121	peller	1.9	if opts.task == 'plot':
122	peller	1.15	Plot_vars= (config.get('Plot_general','List')).split(',')
123	peller	1.8
124	peller	1.12	if not opts.task == 'prep':
125			path = config.get("Directories","samplepath")
126	nmohr	1.21	samplesinfo = config.get("Directories","samplesinfo")
127	nmohr	1.27	info = ParseInfo(samplesinfo,path)
128	peller	1.8
129	peller	1.9	if opts.task == 'plot':
130	peller	1.23	repDict['queue'] = 'all.q'
131	peller	1.9	for item in Plot_vars:
132	peller	1.15	submit(item,repDict)
133	peller	1.9
134	bortigno	1.30
135	peller	1.9	elif opts.task == 'dc':
136	bortigno	1.32	repDict['queue'] = 'short.q'
137	peller	1.9	for item in DC_vars:
138	peller	1.22	if 'ZH%s'%opts.mass in item:
139	peller	1.18	submit(item,repDict)
140	peller	1.22	elif 'ZH' in item and opts.mass == 'all':
141	peller	1.18	submit(item,repDict)
142	peller	1.17
143	peller	1.9	elif opts.task == 'prep':
144			submit('prepare',repDict)
145
146	bortigno	1.30	elif opts.task == 'sys' or opts.task == 'syseval':
147			path = config.get("Directories","SYSin")
148			samplesinfo = config.get("Directories","samplesinfo")
149			info = ParseInfo(samplesinfo,path)
150			if ( opts.samples == ""):
151			for job in info:
152	bortigno	1.31	if (job.subsample): continue #avoid multiple submissions form subsamples
153			# TO FIX FOR SPLITTED SAMPLE
154	bortigno	1.30	submit(job.name,repDict)
155			else:
156			for sample in samplesList:
157			submit(sample,repDict)
158
159			elif opts.task == 'eval':
160			path = config.get("Directories","MVAin")
161			samplesinfo = config.get("Directories","samplesinfo")
162			info = ParseInfo(samplesinfo,path)
163	bortigno	1.10	if ( opts.samples == ""):
164			for job in info:
165	bortigno	1.31	if (job.subsample): continue #avoid multiple submissions from subsamples
166			if(info.checkSplittedSampleName(job.identifier)): # if multiple entries for one name (splitted samples) use the identifier to submit
167			print '@INFO: Splitted samples: submit through identifier'
168			submit(job.identifier,repDict)
169			else: submit(job.name,repDict)
170	bortigno	1.10	else:
171	bortigno	1.11	for sample in samplesList:
172	bortigno	1.10	submit(sample,repDict)
173	peller	1.20
174	bortigno	1.30
175			elif( opts.task == 'split' ):
176			path = config.get("Directories","SPLITin")
177			samplesinfo = config.get("Directories","samplesinfo")
178	bortigno	1.32	repDict['job_id']=opts.nevents_split
179	bortigno	1.30	info = ParseInfo(samplesinfo,path)
180	bortigno	1.32	if ( opts.samples == "" ):
181	bortigno	1.30	for job in info:
182	bortigno	1.32	if (job.subsample): continue #avoid multiple submissions from subsamples
183	bortigno	1.30	submit(job.name,repDict)
184			else:
185			for sample in samplesList:
186			submit(sample,repDict)
187
188			#BDT optimisation
189			elif opts.task == 'mva_opt':
190			total_number_of_steps=1
191			setting = ''
192			for par in (config.get('Optimisation','parameters').split(',')):
193			scan_par=eval(config.get('Optimisation',par))
194			setting+=par+'='+str(scan_par[0])+':'
195			if len(scan_par) > 1 and scan_par[2] != 0:
196			total_number_of_steps+=scan_par[2]
197			setting=setting[:-1] # eliminate last column at the end of the setting string
198			print setting
199			repDict['additional']=setting
200			repDict['job_id']=config.get('Optimisation','training')
201			submit('OPT_main_set',repDict)
202			main_setting=setting
203
204			#Scanning all the parameters found in the training config in the Optimisation sector
205			for par in (config.get('Optimisation','parameters').split(',')):
206			scan_par=eval(config.get('Optimisation',par))
207			print par
208			if len(scan_par) > 1 and scan_par[2] != 0:
209			for step in range(scan_par[2]):
210			value = (scan_par[0])+((1+step)*(scan_par[1]-scan_par[0])/scan_par[2])
211			print value
212			setting=re.sub(par+'.*?:',par+'='+str(value)+':',main_setting)
213			repDict['additional']=setting
214			# repDict['job_id']=config.get('Optimisation','training')
215			submit('OPT_'+par+str(value),repDict)
216			# submit(config.get('Optimisation','training'),repDict)
217			print setting
218
219
220			os.system('qstat')
221			if (opts.philipp_love_progress_bars):
222			os.system('./qstat.py')