VHbb/python/submitThem.py

#! /usr/bin/env python
from optparse import OptionParser
import sys
import time
import os
import shutil

parser = OptionParser()
parser.add_option("-T", "--tag", dest="tag", default="",
                      help="Tag to run the analysis with, example '8TeV' uses config8TeV and pathConfig8TeV to run the analysis")
parser.add_option("-J", "--task", dest="task", default="",
                      help="Task to be done, i.e. 'dc' for Datacards, 'prep' for preparation of Trees, 'plot' to produce plots or 'eval' to write the MVA output or 'sys' to write regression and systematics (or 'syseval' for both). ")
parser.add_option("-M", "--mass", dest="mass", default="125",
                      help="Mass for DC or Plots, 110...135")
parser.add_option("-S","--samples",dest="samples",default="",
                      help="samples you want to run on")
parser.add_option("-F", "--folderTag", dest="ftag", default="",
                      help="Creats a new folder structure for outputs or uses an existing one with the given name")
parser.add_option("-N", "--number-of-events", dest="nevents_split", default=100000,
                      help="Number of events per file when splitting.")
parser.add_option("-P", "--philipp-love-progress-bars", dest="philipp_love_progress_bars", default=False,
                      help="If you share the love of Philipp...")

(opts, args) = parser.parse_args(sys.argv)

import os,shutil,pickle,subprocess,ROOT,re
ROOT.gROOT.SetBatch(True)
from myutils import BetterConfigParser, Sample, ParseInfo, sample_parser
import getpass

if opts.tag == "":
        print "Please provide tag to run the analysis with, example '-T 8TeV' uses config8TeV and pathConfig8TeV to run the analysis."
        sys.exit(123)

if opts.task == "":
    print "Please provide a task.\n-J prep:\tpreparation of Trees\n-J sys:\t\twrite regression and systematics\n-J eval:\tcreate MVA output\n-J plot:\tproduce Plots\n-J dc:\t\twrite workspaces and datacards"
    sys.exit(123)


en = opts.tag

#create the list with the samples to run over
samplesList=opts.samples.split(",")

timestamp = time.asctime().replace(' ','_').replace(':','-')

configs = ['%sconfig/general'%(en),'%sconfig/paths'%(en),'%sconfig/plots'%(en),'%sconfig/training'%(en),'%sconfig/datacards'%(en),'%sconfig/cuts'%(en)]

pathconfig = BetterConfigParser()
pathconfig.read('%sconfig/paths'%(en))

if not opts.ftag == '':
    tagDir = pathconfig.get('Directories','tagDir')
    DirStruct={'tagDir':tagDir,'ftagdir':'%s/%s/'%(tagDir,opts.ftag),'logpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Logs'),'plotpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Plots'),'limitpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Limits'),'confpath':'%s/%s/%s/'%(tagDir,opts.ftag,'config') }

    for keys in ['tagDir','ftagdir','logpath','plotpath','limitpath','confpath']:
        try:
            os.stat(DirStruct[keys])
        except:
            os.mkdir(DirStruct[keys])

    pathfile = open('%sconfig/paths'%(en))
    buffer = pathfile.readlines()
    pathfile.close()
    os.rename('%sconfig/paths'%(en),'%sconfig/paths.bkp'%(en))
    pathfile = open('%sconfig/paths'%(en),'w')
    for line in buffer:
        if line.startswith('plotpath'):
            line = 'plotpath: %s\n'%DirStruct['plotpath']
        elif line.startswith('logpath'):
            line = 'logpath: %s\n'%DirStruct['logpath']
        elif line.startswith('limits'):
            line = 'limits: %s\n'%DirStruct['limitpath']
        pathfile.write(line)
    pathfile.close()

    #copy config files
    for item in configs:
        shutil.copyfile(item,'%s/%s/%s'%(tagDir,opts.ftag,item.strip(en)))


print configs
config = BetterConfigParser()
config.read(configs)

btagLibrary = config.get('BTagReshaping','library')
submitDir = os.getcwd()
os.chdir(os.path.dirname(btagLibrary))
if not os.path.exists(btagLibrary):
    ROOT.gROOT.LoadMacro('%s+'%btagLibrary.replace('_h.so','.h')) 
shutil.copyfile(os.path.basename(btagLibrary),'/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)))
shutil.copyfile('/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)),btagLibrary)
os.chdir(submitDir)
logPath = config.get("Directories","logpath")
logo = open('%s/data/submit.txt' %config.get('Directories','vhbbpath')).readlines()
counter = 0

#check if the logPath exist. If not exit
if( not os.path.isdir(logPath) ):
    print 'ERROR: ' + logPath + ': dir not found.'
    print 'ERROR: Create it before submitting '
    print 'Exit'
    sys.exit(-1)
    

repDict = {'en':en,'logpath':logPath,'job':'','task':opts.task,'queue': 'all.q','timestamp':timestamp,'additional':'','job_id':''}
def submit(job,repDict):
    global counter
    repDict['job'] = job
    nJob = counter % len(logo)
    counter += 1
    if opts.philipp_love_progress_bars:
        repDict['name'] = '"%s"' %logo[nJob].strip()
    else:
        repDict['name'] = '%(job)s_%(en)s%(task)s' %repDict
    command = 'qsub -V -cwd -q %(queue)s -l h_vmem=6G -N %(name)s -o %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.out -e %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task + ' ' + repDict['job_id'] + ' ' + repDict['additional']
    print command
    subprocess.call([command], shell=True)

if opts.task == 'train':
    train_list = (config.get('MVALists','List_for_submitscript')).split(',')
    print train_list
    for item in train_list:
        submit(item,repDict)


if opts.task == 'dc':
    #DC_vars = config.items('Limit')
    DC_vars= (config.get('LimitGeneral','List')).split(',')
    print DC_vars

if opts.task == 'plot':
    Plot_vars= (config.get('Plot_general','List')).split(',')

if not opts.task == 'prep':
    path = config.get("Directories","samplepath")
    samplesinfo = config.get("Directories","samplesinfo")
    info = ParseInfo(samplesinfo,path)

if opts.task == 'plot': 
    repDict['queue'] = 'all.q'
    for item in Plot_vars:
        submit(item,repDict)


elif opts.task == 'dc':
    repDict['queue'] = 'short.q'
    for item in DC_vars:
        if 'ZH%s'%opts.mass in item:
            submit(item,repDict) 
        elif 'ZH' in item and opts.mass == 'all':
            submit(item,repDict)
            
elif opts.task == 'prep':
    if ( opts.samples == ""):
        path = config.get("Directories","PREPin")
        samplesinfo = config.get("Directories","samplesinfo")
        info = ParseInfo(samplesinfo,path)
        for job in info:
            submit(job.name,repDict)

    else:
        for sample in samplesList:
            submit(sample,repDict)
elif opts.task == 'sys' or opts.task == 'syseval':
    path = config.get("Directories","SYSin")
    samplesinfo = config.get("Directories","samplesinfo")
    info = ParseInfo(samplesinfo,path)
    if ( opts.samples == ""):
        for job in info:
            if (job.subsample): 
                continue #avoid multiple submissions form subsamples
            # TO FIX FOR SPLITTED SAMPLE
            submit(job.name,repDict)
    else:
        for sample in samplesList:
            submit(sample,repDict)

elif opts.task == 'eval':
    path = config.get("Directories","MVAin")
    samplesinfo = config.get("Directories","samplesinfo")
    info = ParseInfo(samplesinfo,path)
    if ( opts.samples == ""):
        for job in info:
            if (job.subsample): 
                continue #avoid multiple submissions from subsamples
            if(info.checkSplittedSampleName(job.identifier)): # if multiple entries for one name  (splitted samples) use the identifier to submit
                print '@INFO: Splitted samples: submit through identifier'
                submit(job.identifier,repDict)
            else: submit(job.name,repDict)
    else:
        for sample in samplesList:
            submit(sample,repDict)


elif( opts.task == 'split' ):
        path = config.get("Directories","SPLITin")
        samplesinfo = config.get("Directories","samplesinfo")
        repDict['job_id']=opts.nevents_split
        info = ParseInfo(samplesinfo,path)
        if ( opts.samples == "" ):
                for job in info:
                        if (job.subsample): continue #avoid multiple submissions from subsamples
                        submit(job.name,repDict)
        else:
                for sample in samplesList:
                        submit(sample,repDict)

#BDT optimisation
elif opts.task == 'mva_opt':
        total_number_of_steps=1
        setting = ''
        for par in (config.get('Optimisation','parameters').split(',')):
                scan_par=eval(config.get('Optimisation',par))
                setting+=par+'='+str(scan_par[0])+':'
                if len(scan_par) > 1 and scan_par[2] != 0:
                        total_number_of_steps+=scan_par[2]
        setting=setting[:-1] # eliminate last column at the end of the setting string
        print setting
        repDict['additional']=setting
        repDict['job_id']=config.get('Optimisation','training')
        submit('OPT_main_set',repDict)
        main_setting=setting

        #Scanning all the parameters found in the training config in the Optimisation sector
        for par in (config.get('Optimisation','parameters').split(',')):
                scan_par=eval(config.get('Optimisation',par))
                print par
                if len(scan_par) > 1 and scan_par[2] != 0:
                        for step in range(scan_par[2]):
                                value = (scan_par[0])+((1+step)*(scan_par[1]-scan_par[0])/scan_par[2])
                                print value
                                setting=re.sub(par+'.*?:',par+'='+str(value)+':',main_setting)
                                repDict['additional']=setting
#                               repDict['job_id']=config.get('Optimisation','training')
                                submit('OPT_'+par+str(value),repDict)
#                               submit(config.get('Optimisation','training'),repDict)
                                print setting


os.system('qstat')
if (opts.philipp_love_progress_bars):
        os.system('./qstat.py') 
Revision:	1.34
Committed:	Thu Feb 28 16:59:09 2013 UTC (12 years, 2 months ago) by nmohr
Content type:	text/x-python
Branch:	MAIN
Changes since 1.33:	+28 -15 lines
Log Message:	Gimmick of the week
#	User	Rev	Content
1	nmohr	1.1	#! /usr/bin/env python
2	nmohr	1.6	from optparse import OptionParser
3	bortigno	1.26	import sys
4	peller	1.28	import time
5			import os
6			import shutil
7	nmohr	1.7
8	nmohr	1.6	parser = OptionParser()
9			parser.add_option("-T", "--tag", dest="tag", default="",
10			help="Tag to run the analysis with, example '8TeV' uses config8TeV and pathConfig8TeV to run the analysis")
11	peller	1.9	parser.add_option("-J", "--task", dest="task", default="",
12	nmohr	1.19	help="Task to be done, i.e. 'dc' for Datacards, 'prep' for preparation of Trees, 'plot' to produce plots or 'eval' to write the MVA output or 'sys' to write regression and systematics (or 'syseval' for both). ")
13	peller	1.9	parser.add_option("-M", "--mass", dest="mass", default="125",
14	bortigno	1.10	help="Mass for DC or Plots, 110...135")
15			parser.add_option("-S","--samples",dest="samples",default="",
16			help="samples you want to run on")
17	peller	1.28	parser.add_option("-F", "--folderTag", dest="ftag", default="",
18			help="Creats a new folder structure for outputs or uses an existing one with the given name")
19	bortigno	1.30	parser.add_option("-N", "--number-of-events", dest="nevents_split", default=100000,
20			help="Number of events per file when splitting.")
21			parser.add_option("-P", "--philipp-love-progress-bars", dest="philipp_love_progress_bars", default=False,
22			help="If you share the love of Philipp...")
23
24	bortigno	1.26	(opts, args) = parser.parse_args(sys.argv)
25
26	bortigno	1.30	import os,shutil,pickle,subprocess,ROOT,re
27	nmohr	1.27	ROOT.gROOT.SetBatch(True)
28	bortigno	1.31	from myutils import BetterConfigParser, Sample, ParseInfo, sample_parser
29	bortigno	1.26	import getpass
30	nmohr	1.6
31			if opts.tag == "":
32			print "Please provide tag to run the analysis with, example '-T 8TeV' uses config8TeV and pathConfig8TeV to run the analysis."
33			sys.exit(123)
34	peller	1.9
35			if opts.task == "":
36			print "Please provide a task.\n-J prep:\tpreparation of Trees\n-J sys:\t\twrite regression and systematics\n-J eval:\tcreate MVA output\n-J plot:\tproduce Plots\n-J dc:\t\twrite workspaces and datacards"
37			sys.exit(123)
38
39	peller	1.28
40			en = opts.tag
41
42	bortigno	1.11	#create the list with the samples to run over
43			samplesList=opts.samples.split(",")
44	bortigno	1.10
45	peller	1.28	timestamp = time.asctime().replace(' ','_').replace(':','-')
46
47	nmohr	1.25	configs = ['%sconfig/general'%(en),'%sconfig/paths'%(en),'%sconfig/plots'%(en),'%sconfig/training'%(en),'%sconfig/datacards'%(en),'%sconfig/cuts'%(en)]
48	peller	1.28
49			pathconfig = BetterConfigParser()
50			pathconfig.read('%sconfig/paths'%(en))
51
52			if not opts.ftag == '':
53			tagDir = pathconfig.get('Directories','tagDir')
54			DirStruct={'tagDir':tagDir,'ftagdir':'%s/%s/'%(tagDir,opts.ftag),'logpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Logs'),'plotpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Plots'),'limitpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Limits'),'confpath':'%s/%s/%s/'%(tagDir,opts.ftag,'config') }
55
56			for keys in ['tagDir','ftagdir','logpath','plotpath','limitpath','confpath']:
57			try:
58			os.stat(DirStruct[keys])
59			except:
60			os.mkdir(DirStruct[keys])
61
62			pathfile = open('%sconfig/paths'%(en))
63			buffer = pathfile.readlines()
64			pathfile.close()
65			os.rename('%sconfig/paths'%(en),'%sconfig/paths.bkp'%(en))
66			pathfile = open('%sconfig/paths'%(en),'w')
67			for line in buffer:
68			if line.startswith('plotpath'):
69			line = 'plotpath: %s\n'%DirStruct['plotpath']
70			elif line.startswith('logpath'):
71			line = 'logpath: %s\n'%DirStruct['logpath']
72			elif line.startswith('limits'):
73			line = 'limits: %s\n'%DirStruct['limitpath']
74			pathfile.write(line)
75			pathfile.close()
76
77			#copy config files
78			for item in configs:
79			shutil.copyfile(item,'%s/%s/%s'%(tagDir,opts.ftag,item.strip(en)))
80
81
82	nmohr	1.6	print configs
83			config = BetterConfigParser()
84			config.read(configs)
85	peller	1.28
86	nmohr	1.7	btagLibrary = config.get('BTagReshaping','library')
87			submitDir = os.getcwd()
88			os.chdir(os.path.dirname(btagLibrary))
89			if not os.path.exists(btagLibrary):
90			ROOT.gROOT.LoadMacro('%s+'%btagLibrary.replace('_h.so','.h'))
91			shutil.copyfile(os.path.basename(btagLibrary),'/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)))
92			shutil.copyfile('/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)),btagLibrary)
93			os.chdir(submitDir)
94	nmohr	1.6	logPath = config.get("Directories","logpath")
95	nmohr	1.34	logo = open('%s/data/submit.txt' %config.get('Directories','vhbbpath')).readlines()
96			counter = 0
97
98	bortigno	1.14	#check if the logPath exist. If not exit
99			if( not os.path.isdir(logPath) ):
100	nmohr	1.34	print 'ERROR: ' + logPath + ': dir not found.'
101			print 'ERROR: Create it before submitting '
102			print 'Exit'
103			sys.exit(-1)
104
105	bortigno	1.14
106	bortigno	1.30	repDict = {'en':en,'logpath':logPath,'job':'','task':opts.task,'queue': 'all.q','timestamp':timestamp,'additional':'','job_id':''}
107	nmohr	1.6	def submit(job,repDict):
108	nmohr	1.34	global counter
109			repDict['job'] = job
110			nJob = counter % len(logo)
111			counter += 1
112			if opts.philipp_love_progress_bars:
113			repDict['name'] = '"%s"' %logo[nJob].strip()
114			else:
115			repDict['name'] = '%(job)s_%(en)s%(task)s' %repDict
116			command = 'qsub -V -cwd -q %(queue)s -l h_vmem=6G -N %(name)s -o %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.out -e %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task + ' ' + repDict['job_id'] + ' ' + repDict['additional']
117			print command
118			subprocess.call([command], shell=True)
119	nmohr	1.1
120	peller	1.29	if opts.task == 'train':
121			train_list = (config.get('MVALists','List_for_submitscript')).split(',')
122			print train_list
123			for item in train_list:
124			submit(item,repDict)
125
126
127	peller	1.9	if opts.task == 'dc':
128	peller	1.18	#DC_vars = config.items('Limit')
129			DC_vars= (config.get('LimitGeneral','List')).split(',')
130			print DC_vars
131
132	peller	1.9	if opts.task == 'plot':
133	peller	1.15	Plot_vars= (config.get('Plot_general','List')).split(',')
134	peller	1.8
135	peller	1.12	if not opts.task == 'prep':
136			path = config.get("Directories","samplepath")
137	nmohr	1.21	samplesinfo = config.get("Directories","samplesinfo")
138	nmohr	1.27	info = ParseInfo(samplesinfo,path)
139	peller	1.8
140	peller	1.9	if opts.task == 'plot':
141	peller	1.23	repDict['queue'] = 'all.q'
142	peller	1.9	for item in Plot_vars:
143	peller	1.15	submit(item,repDict)
144	peller	1.9
145	bortigno	1.30
146	peller	1.9	elif opts.task == 'dc':
147	bortigno	1.32	repDict['queue'] = 'short.q'
148	peller	1.9	for item in DC_vars:
149	peller	1.22	if 'ZH%s'%opts.mass in item:
150	peller	1.18	submit(item,repDict)
151	peller	1.22	elif 'ZH' in item and opts.mass == 'all':
152	peller	1.18	submit(item,repDict)
153	peller	1.17
154	peller	1.9	elif opts.task == 'prep':
155	nmohr	1.33	if ( opts.samples == ""):
156			path = config.get("Directories","PREPin")
157			samplesinfo = config.get("Directories","samplesinfo")
158			info = ParseInfo(samplesinfo,path)
159			for job in info:
160			submit(job.name,repDict)
161	peller	1.9
162	nmohr	1.33	else:
163			for sample in samplesList:
164			submit(sample,repDict)
165	bortigno	1.30	elif opts.task == 'sys' or opts.task == 'syseval':
166			path = config.get("Directories","SYSin")
167			samplesinfo = config.get("Directories","samplesinfo")
168			info = ParseInfo(samplesinfo,path)
169			if ( opts.samples == ""):
170			for job in info:
171	nmohr	1.34	if (job.subsample):
172			continue #avoid multiple submissions form subsamples
173			# TO FIX FOR SPLITTED SAMPLE
174	bortigno	1.30	submit(job.name,repDict)
175			else:
176			for sample in samplesList:
177			submit(sample,repDict)
178
179			elif opts.task == 'eval':
180			path = config.get("Directories","MVAin")
181			samplesinfo = config.get("Directories","samplesinfo")
182			info = ParseInfo(samplesinfo,path)
183	bortigno	1.10	if ( opts.samples == ""):
184			for job in info:
185	nmohr	1.34	if (job.subsample):
186			continue #avoid multiple submissions from subsamples
187			if(info.checkSplittedSampleName(job.identifier)): # if multiple entries for one name (splitted samples) use the identifier to submit
188			print '@INFO: Splitted samples: submit through identifier'
189			submit(job.identifier,repDict)
190			else: submit(job.name,repDict)
191	bortigno	1.10	else:
192	bortigno	1.11	for sample in samplesList:
193	bortigno	1.10	submit(sample,repDict)
194	peller	1.20
195	bortigno	1.30
196			elif( opts.task == 'split' ):
197			path = config.get("Directories","SPLITin")
198			samplesinfo = config.get("Directories","samplesinfo")
199	bortigno	1.32	repDict['job_id']=opts.nevents_split
200	bortigno	1.30	info = ParseInfo(samplesinfo,path)
201	bortigno	1.32	if ( opts.samples == "" ):
202	bortigno	1.30	for job in info:
203	bortigno	1.32	if (job.subsample): continue #avoid multiple submissions from subsamples
204	bortigno	1.30	submit(job.name,repDict)
205			else:
206			for sample in samplesList:
207			submit(sample,repDict)
208
209			#BDT optimisation
210			elif opts.task == 'mva_opt':
211			total_number_of_steps=1
212			setting = ''
213			for par in (config.get('Optimisation','parameters').split(',')):
214			scan_par=eval(config.get('Optimisation',par))
215			setting+=par+'='+str(scan_par[0])+':'
216			if len(scan_par) > 1 and scan_par[2] != 0:
217			total_number_of_steps+=scan_par[2]
218			setting=setting[:-1] # eliminate last column at the end of the setting string
219			print setting
220			repDict['additional']=setting
221			repDict['job_id']=config.get('Optimisation','training')
222			submit('OPT_main_set',repDict)
223			main_setting=setting
224
225			#Scanning all the parameters found in the training config in the Optimisation sector
226			for par in (config.get('Optimisation','parameters').split(',')):
227			scan_par=eval(config.get('Optimisation',par))
228			print par
229			if len(scan_par) > 1 and scan_par[2] != 0:
230			for step in range(scan_par[2]):
231			value = (scan_par[0])+((1+step)*(scan_par[1]-scan_par[0])/scan_par[2])
232			print value
233			setting=re.sub(par+'.*?:',par+'='+str(value)+':',main_setting)
234			repDict['additional']=setting
235			# repDict['job_id']=config.get('Optimisation','training')
236			submit('OPT_'+par+str(value),repDict)
237			# submit(config.get('Optimisation','training'),repDict)
238			print setting
239
240
241			os.system('qstat')
242			if (opts.philipp_love_progress_bars):
243			os.system('./qstat.py')