VHbb/python/submitThem.py

#! /usr/bin/env python
from optparse import OptionParser
import sys
import time
import os
import shutil

parser = OptionParser()
parser.add_option("-T", "--tag", dest="tag", default="",
                      help="Tag to run the analysis with, example '8TeV' uses config8TeV and pathConfig8TeV to run the analysis")
parser.add_option("-J", "--task", dest="task", default="",
                      help="Task to be done, i.e. 'dc' for Datacards, 'prep' for preparation of Trees, 'plot' to produce plots or 'eval' to write the MVA output or 'sys' to write regression and systematics (or 'syseval' for both). ")
parser.add_option("-M", "--mass", dest="mass", default="125",
                      help="Mass for DC or Plots, 110...135")
parser.add_option("-S","--samples",dest="samples",default="",
                      help="samples you want to run on")
parser.add_option("-F", "--folderTag", dest="ftag", default="",
                      help="Creats a new folder structure for outputs or uses an existing one with the given name")
parser.add_option("-N", "--number-of-events", dest="nevents_split", default=100000,
                      help="Number of events per file when splitting.")
parser.add_option("-P", "--philipp-love-progress-bars", dest="philipp_love_progress_bars", default=False,
                      help="If you share the love of Philipp...")

(opts, args) = parser.parse_args(sys.argv)

import os,shutil,pickle,subprocess,ROOT,re
ROOT.gROOT.SetBatch(True)
from myutils import BetterConfigParser, Sample, ParseInfo, sample_parser
import getpass

if opts.tag == "":
        print "Please provide tag to run the analysis with, example '-T 8TeV' uses config8TeV and pathConfig8TeV to run the analysis."
        sys.exit(123)

if opts.task == "":
    print "Please provide a task.\n-J prep:\tpreparation of Trees\n-J sys:\t\twrite regression and systematics\n-J eval:\tcreate MVA output\n-J plot:\tproduce Plots\n-J dc:\t\twrite workspaces and datacards"
    sys.exit(123)


en = opts.tag

#create the list with the samples to run over
samplesList=opts.samples.split(",")

timestamp = time.asctime().replace(' ','_').replace(':','-')

configs = ['%sconfig/general'%(en),'%sconfig/paths'%(en),'%sconfig/plots'%(en),'%sconfig/training'%(en),'%sconfig/datacards'%(en),'%sconfig/cuts'%(en)]

pathconfig = BetterConfigParser()
pathconfig.read('%sconfig/paths'%(en))

if not opts.ftag == '':
    tagDir = pathconfig.get('Directories','tagDir')
    DirStruct={'tagDir':tagDir,'ftagdir':'%s/%s/'%(tagDir,opts.ftag),'logpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Logs'),'plotpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Plots'),'limitpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Limits'),'confpath':'%s/%s/%s/'%(tagDir,opts.ftag,'config') }

    for keys in ['tagDir','ftagdir','logpath','plotpath','limitpath','confpath']:
        try:
            os.stat(DirStruct[keys])
        except:
            os.mkdir(DirStruct[keys])

    pathfile = open('%sconfig/paths'%(en))
    buffer = pathfile.readlines()
    pathfile.close()
    os.rename('%sconfig/paths'%(en),'%sconfig/paths.bkp'%(en))
    pathfile = open('%sconfig/paths'%(en),'w')
    for line in buffer:
        if line.startswith('plotpath'):
            line = 'plotpath: %s\n'%DirStruct['plotpath']
        elif line.startswith('logpath'):
            line = 'logpath: %s\n'%DirStruct['logpath']
        elif line.startswith('limits'):
            line = 'limits: %s\n'%DirStruct['limitpath']
        pathfile.write(line)
    pathfile.close()

    #copy config files
    for item in configs:
        shutil.copyfile(item,'%s/%s/%s'%(tagDir,opts.ftag,item.strip(en)))


print configs
config = BetterConfigParser()
config.read(configs)

btagLibrary = config.get('BTagReshaping','library')
submitDir = os.getcwd()
os.chdir(os.path.dirname(btagLibrary))
if not os.path.exists(btagLibrary):
    ROOT.gROOT.LoadMacro('%s+'%btagLibrary.replace('_h.so','.h')) 
shutil.copyfile(os.path.basename(btagLibrary),'/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)))
shutil.copyfile('/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)),btagLibrary)
os.chdir(submitDir)
logPath = config.get("Directories","logpath")
#check if the logPath exist. If not exit
if( not os.path.isdir(logPath) ):
        print 'ERROR: ' + logPath + ': dir not found.'
        print 'ERROR: Create it before submitting '
        print 'Exit'
        sys.exit(-1)

repDict = {'en':en,'logpath':logPath,'job':'','task':opts.task,'queue': 'all.q','timestamp':timestamp,'additional':'','job_id':''}
def submit(job,repDict):
        repDict['job'] = job
        command = 'qsub -V -cwd -q %(queue)s -l h_vmem=6G -N %(job)s_%(en)s%(task)s -o %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.out -e %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task + ' ' + repDict['job_id'] + ' ' + repDict['additional']
        print command
        subprocess.call([command], shell=True)

if opts.task == 'train':
    train_list = (config.get('MVALists','List_for_submitscript')).split(',')
    print train_list
    for item in train_list:
        submit(item,repDict)


if opts.task == 'dc':
    #DC_vars = config.items('Limit')
    DC_vars= (config.get('LimitGeneral','List')).split(',')
    print DC_vars

if opts.task == 'plot':
    Plot_vars= (config.get('Plot_general','List')).split(',')

if not opts.task == 'prep':
    path = config.get("Directories","samplepath")
    samplesinfo = config.get("Directories","samplesinfo")
    info = ParseInfo(samplesinfo,path)

if opts.task == 'plot': 
    repDict['queue'] = 'all.q'
    for item in Plot_vars:
        submit(item,repDict)


elif opts.task == 'dc':
    repDict['queue'] = 'short.q'
    for item in DC_vars:
        if 'ZH%s'%opts.mass in item:
            submit(item,repDict) 
        elif 'ZH' in item and opts.mass == 'all':
            submit(item,repDict)
            
elif opts.task == 'prep':
    if ( opts.samples == ""):
        path = config.get("Directories","PREPin")
        samplesinfo = config.get("Directories","samplesinfo")
        info = ParseInfo(samplesinfo,path)
        for job in info:
            submit(job.name,repDict)

    else:
        for sample in samplesList:
            submit(sample,repDict)
elif opts.task == 'sys' or opts.task == 'syseval':
    path = config.get("Directories","SYSin")
    samplesinfo = config.get("Directories","samplesinfo")
    info = ParseInfo(samplesinfo,path)
    if ( opts.samples == ""):
        for job in info:
            if (job.subsample): continue #avoid multiple submissions form subsamples
            # TO FIX FOR SPLITTED SAMPLE
            submit(job.name,repDict)
    else:
        for sample in samplesList:
            submit(sample,repDict)

elif opts.task == 'eval':
    path = config.get("Directories","MVAin")
    samplesinfo = config.get("Directories","samplesinfo")
    info = ParseInfo(samplesinfo,path)
    if ( opts.samples == ""):
        for job in info:
            if (job.subsample): continue #avoid multiple submissions from subsamples
            if(info.checkSplittedSampleName(job.identifier)): # if multiple entries for one name  (splitted samples) use the identifier to submit
                    print '@INFO: Splitted samples: submit through identifier'
                    submit(job.identifier,repDict)
            else: submit(job.name,repDict)
    else:
        for sample in samplesList:
            submit(sample,repDict)


elif( opts.task == 'split' ):
        path = config.get("Directories","SPLITin")
        samplesinfo = config.get("Directories","samplesinfo")
        repDict['job_id']=opts.nevents_split
        info = ParseInfo(samplesinfo,path)
        if ( opts.samples == "" ):
                for job in info:
                        if (job.subsample): continue #avoid multiple submissions from subsamples
                        submit(job.name,repDict)
        else:
                for sample in samplesList:
                        submit(sample,repDict)

#BDT optimisation
elif opts.task == 'mva_opt':
        total_number_of_steps=1
        setting = ''
        for par in (config.get('Optimisation','parameters').split(',')):
                scan_par=eval(config.get('Optimisation',par))
                setting+=par+'='+str(scan_par[0])+':'
                if len(scan_par) > 1 and scan_par[2] != 0:
                        total_number_of_steps+=scan_par[2]
        setting=setting[:-1] # eliminate last column at the end of the setting string
        print setting
        repDict['additional']=setting
        repDict['job_id']=config.get('Optimisation','training')
        submit('OPT_main_set',repDict)
        main_setting=setting

        #Scanning all the parameters found in the training config in the Optimisation sector
        for par in (config.get('Optimisation','parameters').split(',')):
                scan_par=eval(config.get('Optimisation',par))
                print par
                if len(scan_par) > 1 and scan_par[2] != 0:
                        for step in range(scan_par[2]):
                                value = (scan_par[0])+((1+step)*(scan_par[1]-scan_par[0])/scan_par[2])
                                print value
                                setting=re.sub(par+'.*?:',par+'='+str(value)+':',main_setting)
                                repDict['additional']=setting
#                               repDict['job_id']=config.get('Optimisation','training')
                                submit('OPT_'+par+str(value),repDict)
#                               submit(config.get('Optimisation','training'),repDict)
                                print setting


os.system('qstat')
if (opts.philipp_love_progress_bars):
        os.system('./qstat.py') 
Revision:	1.33
Committed:	Thu Feb 28 16:38:49 2013 UTC (12 years, 2 months ago) by nmohr
Content type:	text/x-python
Branch:	MAIN
Changes since 1.32:	+9 -1 lines
Log Message:	Prepare split jobs
#	User	Rev	Content
1	nmohr	1.1	#! /usr/bin/env python
2	nmohr	1.6	from optparse import OptionParser
3	bortigno	1.26	import sys
4	peller	1.28	import time
5			import os
6			import shutil
7	nmohr	1.7
8	nmohr	1.6	parser = OptionParser()
9			parser.add_option("-T", "--tag", dest="tag", default="",
10			help="Tag to run the analysis with, example '8TeV' uses config8TeV and pathConfig8TeV to run the analysis")
11	peller	1.9	parser.add_option("-J", "--task", dest="task", default="",
12	nmohr	1.19	help="Task to be done, i.e. 'dc' for Datacards, 'prep' for preparation of Trees, 'plot' to produce plots or 'eval' to write the MVA output or 'sys' to write regression and systematics (or 'syseval' for both). ")
13	peller	1.9	parser.add_option("-M", "--mass", dest="mass", default="125",
14	bortigno	1.10	help="Mass for DC or Plots, 110...135")
15			parser.add_option("-S","--samples",dest="samples",default="",
16			help="samples you want to run on")
17	peller	1.28	parser.add_option("-F", "--folderTag", dest="ftag", default="",
18			help="Creats a new folder structure for outputs or uses an existing one with the given name")
19	bortigno	1.30	parser.add_option("-N", "--number-of-events", dest="nevents_split", default=100000,
20			help="Number of events per file when splitting.")
21			parser.add_option("-P", "--philipp-love-progress-bars", dest="philipp_love_progress_bars", default=False,
22			help="If you share the love of Philipp...")
23
24	bortigno	1.26	(opts, args) = parser.parse_args(sys.argv)
25
26	bortigno	1.30	import os,shutil,pickle,subprocess,ROOT,re
27	nmohr	1.27	ROOT.gROOT.SetBatch(True)
28	bortigno	1.31	from myutils import BetterConfigParser, Sample, ParseInfo, sample_parser
29	bortigno	1.26	import getpass
30	nmohr	1.6
31			if opts.tag == "":
32			print "Please provide tag to run the analysis with, example '-T 8TeV' uses config8TeV and pathConfig8TeV to run the analysis."
33			sys.exit(123)
34	peller	1.9
35			if opts.task == "":
36			print "Please provide a task.\n-J prep:\tpreparation of Trees\n-J sys:\t\twrite regression and systematics\n-J eval:\tcreate MVA output\n-J plot:\tproduce Plots\n-J dc:\t\twrite workspaces and datacards"
37			sys.exit(123)
38
39	peller	1.28
40			en = opts.tag
41
42	bortigno	1.11	#create the list with the samples to run over
43			samplesList=opts.samples.split(",")
44	bortigno	1.10
45	peller	1.28	timestamp = time.asctime().replace(' ','_').replace(':','-')
46
47	nmohr	1.25	configs = ['%sconfig/general'%(en),'%sconfig/paths'%(en),'%sconfig/plots'%(en),'%sconfig/training'%(en),'%sconfig/datacards'%(en),'%sconfig/cuts'%(en)]
48	peller	1.28
49			pathconfig = BetterConfigParser()
50			pathconfig.read('%sconfig/paths'%(en))
51
52			if not opts.ftag == '':
53			tagDir = pathconfig.get('Directories','tagDir')
54			DirStruct={'tagDir':tagDir,'ftagdir':'%s/%s/'%(tagDir,opts.ftag),'logpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Logs'),'plotpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Plots'),'limitpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Limits'),'confpath':'%s/%s/%s/'%(tagDir,opts.ftag,'config') }
55
56			for keys in ['tagDir','ftagdir','logpath','plotpath','limitpath','confpath']:
57			try:
58			os.stat(DirStruct[keys])
59			except:
60			os.mkdir(DirStruct[keys])
61
62			pathfile = open('%sconfig/paths'%(en))
63			buffer = pathfile.readlines()
64			pathfile.close()
65			os.rename('%sconfig/paths'%(en),'%sconfig/paths.bkp'%(en))
66			pathfile = open('%sconfig/paths'%(en),'w')
67			for line in buffer:
68			if line.startswith('plotpath'):
69			line = 'plotpath: %s\n'%DirStruct['plotpath']
70			elif line.startswith('logpath'):
71			line = 'logpath: %s\n'%DirStruct['logpath']
72			elif line.startswith('limits'):
73			line = 'limits: %s\n'%DirStruct['limitpath']
74			pathfile.write(line)
75			pathfile.close()
76
77			#copy config files
78			for item in configs:
79			shutil.copyfile(item,'%s/%s/%s'%(tagDir,opts.ftag,item.strip(en)))
80
81
82	nmohr	1.6	print configs
83			config = BetterConfigParser()
84			config.read(configs)
85	peller	1.28
86	nmohr	1.7	btagLibrary = config.get('BTagReshaping','library')
87			submitDir = os.getcwd()
88			os.chdir(os.path.dirname(btagLibrary))
89			if not os.path.exists(btagLibrary):
90			ROOT.gROOT.LoadMacro('%s+'%btagLibrary.replace('_h.so','.h'))
91			shutil.copyfile(os.path.basename(btagLibrary),'/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)))
92			shutil.copyfile('/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)),btagLibrary)
93			os.chdir(submitDir)
94	nmohr	1.6	logPath = config.get("Directories","logpath")
95	bortigno	1.14	#check if the logPath exist. If not exit
96			if( not os.path.isdir(logPath) ):
97			print 'ERROR: ' + logPath + ': dir not found.'
98			print 'ERROR: Create it before submitting '
99			print 'Exit'
100			sys.exit(-1)
101
102	bortigno	1.30	repDict = {'en':en,'logpath':logPath,'job':'','task':opts.task,'queue': 'all.q','timestamp':timestamp,'additional':'','job_id':''}
103	nmohr	1.6	def submit(job,repDict):
104			repDict['job'] = job
105	bortigno	1.30	command = 'qsub -V -cwd -q %(queue)s -l h_vmem=6G -N %(job)s_%(en)s%(task)s -o %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.out -e %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task + ' ' + repDict['job_id'] + ' ' + repDict['additional']
106	nmohr	1.1	print command
107			subprocess.call([command], shell=True)
108
109	peller	1.29	if opts.task == 'train':
110			train_list = (config.get('MVALists','List_for_submitscript')).split(',')
111			print train_list
112			for item in train_list:
113			submit(item,repDict)
114
115
116	peller	1.9	if opts.task == 'dc':
117	peller	1.18	#DC_vars = config.items('Limit')
118			DC_vars= (config.get('LimitGeneral','List')).split(',')
119			print DC_vars
120
121	peller	1.9	if opts.task == 'plot':
122	peller	1.15	Plot_vars= (config.get('Plot_general','List')).split(',')
123	peller	1.8
124	peller	1.12	if not opts.task == 'prep':
125			path = config.get("Directories","samplepath")
126	nmohr	1.21	samplesinfo = config.get("Directories","samplesinfo")
127	nmohr	1.27	info = ParseInfo(samplesinfo,path)
128	peller	1.8
129	peller	1.9	if opts.task == 'plot':
130	peller	1.23	repDict['queue'] = 'all.q'
131	peller	1.9	for item in Plot_vars:
132	peller	1.15	submit(item,repDict)
133	peller	1.9
134	bortigno	1.30
135	peller	1.9	elif opts.task == 'dc':
136	bortigno	1.32	repDict['queue'] = 'short.q'
137	peller	1.9	for item in DC_vars:
138	peller	1.22	if 'ZH%s'%opts.mass in item:
139	peller	1.18	submit(item,repDict)
140	peller	1.22	elif 'ZH' in item and opts.mass == 'all':
141	peller	1.18	submit(item,repDict)
142	peller	1.17
143	peller	1.9	elif opts.task == 'prep':
144	nmohr	1.33	if ( opts.samples == ""):
145			path = config.get("Directories","PREPin")
146			samplesinfo = config.get("Directories","samplesinfo")
147			info = ParseInfo(samplesinfo,path)
148			for job in info:
149			submit(job.name,repDict)
150	peller	1.9
151	nmohr	1.33	else:
152			for sample in samplesList:
153			submit(sample,repDict)
154	bortigno	1.30	elif opts.task == 'sys' or opts.task == 'syseval':
155			path = config.get("Directories","SYSin")
156			samplesinfo = config.get("Directories","samplesinfo")
157			info = ParseInfo(samplesinfo,path)
158			if ( opts.samples == ""):
159			for job in info:
160	bortigno	1.31	if (job.subsample): continue #avoid multiple submissions form subsamples
161			# TO FIX FOR SPLITTED SAMPLE
162	bortigno	1.30	submit(job.name,repDict)
163			else:
164			for sample in samplesList:
165			submit(sample,repDict)
166
167			elif opts.task == 'eval':
168			path = config.get("Directories","MVAin")
169			samplesinfo = config.get("Directories","samplesinfo")
170			info = ParseInfo(samplesinfo,path)
171	bortigno	1.10	if ( opts.samples == ""):
172			for job in info:
173	bortigno	1.31	if (job.subsample): continue #avoid multiple submissions from subsamples
174			if(info.checkSplittedSampleName(job.identifier)): # if multiple entries for one name (splitted samples) use the identifier to submit
175			print '@INFO: Splitted samples: submit through identifier'
176			submit(job.identifier,repDict)
177			else: submit(job.name,repDict)
178	bortigno	1.10	else:
179	bortigno	1.11	for sample in samplesList:
180	bortigno	1.10	submit(sample,repDict)
181	peller	1.20
182	bortigno	1.30
183			elif( opts.task == 'split' ):
184			path = config.get("Directories","SPLITin")
185			samplesinfo = config.get("Directories","samplesinfo")
186	bortigno	1.32	repDict['job_id']=opts.nevents_split
187	bortigno	1.30	info = ParseInfo(samplesinfo,path)
188	bortigno	1.32	if ( opts.samples == "" ):
189	bortigno	1.30	for job in info:
190	bortigno	1.32	if (job.subsample): continue #avoid multiple submissions from subsamples
191	bortigno	1.30	submit(job.name,repDict)
192			else:
193			for sample in samplesList:
194			submit(sample,repDict)
195
196			#BDT optimisation
197			elif opts.task == 'mva_opt':
198			total_number_of_steps=1
199			setting = ''
200			for par in (config.get('Optimisation','parameters').split(',')):
201			scan_par=eval(config.get('Optimisation',par))
202			setting+=par+'='+str(scan_par[0])+':'
203			if len(scan_par) > 1 and scan_par[2] != 0:
204			total_number_of_steps+=scan_par[2]
205			setting=setting[:-1] # eliminate last column at the end of the setting string
206			print setting
207			repDict['additional']=setting
208			repDict['job_id']=config.get('Optimisation','training')
209			submit('OPT_main_set',repDict)
210			main_setting=setting
211
212			#Scanning all the parameters found in the training config in the Optimisation sector
213			for par in (config.get('Optimisation','parameters').split(',')):
214			scan_par=eval(config.get('Optimisation',par))
215			print par
216			if len(scan_par) > 1 and scan_par[2] != 0:
217			for step in range(scan_par[2]):
218			value = (scan_par[0])+((1+step)*(scan_par[1]-scan_par[0])/scan_par[2])
219			print value
220			setting=re.sub(par+'.*?:',par+'='+str(value)+':',main_setting)
221			repDict['additional']=setting
222			# repDict['job_id']=config.get('Optimisation','training')
223			submit('OPT_'+par+str(value),repDict)
224			# submit(config.get('Optimisation','training'),repDict)
225			print setting
226
227
228			os.system('qstat')
229			if (opts.philipp_love_progress_bars):
230			os.system('./qstat.py')