| 1 |
|
#! /usr/bin/env python |
| 2 |
– |
import os,shutil,sys,pickle,subprocess,ROOT |
| 2 |
|
from optparse import OptionParser |
| 3 |
< |
from BetterConfigParser import BetterConfigParser |
| 4 |
< |
from samplesclass import sample |
| 5 |
< |
import getpass |
| 3 |
> |
import sys |
| 4 |
> |
import time |
| 5 |
> |
import os |
| 6 |
> |
import shutil |
| 7 |
|
|
| 8 |
|
parser = OptionParser() |
| 9 |
|
parser.add_option("-T", "--tag", dest="tag", default="", |
| 10 |
|
help="Tag to run the analysis with, example '8TeV' uses config8TeV and pathConfig8TeV to run the analysis") |
| 11 |
|
parser.add_option("-J", "--task", dest="task", default="", |
| 12 |
< |
help="Task to be done, i.e. 'dc' for Datacards, 'prep' for preparation of Trees, 'plot' to produce plots or 'eval' to write the MVA output or 'sys' to write regression and systematics. ") |
| 12 |
> |
help="Task to be done, i.e. 'dc' for Datacards, 'prep' for preparation of Trees, 'plot' to produce plots or 'eval' to write the MVA output or 'sys' to write regression and systematics (or 'syseval' for both). ") |
| 13 |
|
parser.add_option("-M", "--mass", dest="mass", default="125", |
| 14 |
|
help="Mass for DC or Plots, 110...135") |
| 15 |
|
parser.add_option("-S","--samples",dest="samples",default="", |
| 16 |
|
help="samples you want to run on") |
| 17 |
< |
|
| 17 |
> |
parser.add_option("-F", "--folderTag", dest="ftag", default="", |
| 18 |
> |
help="Creats a new folder structure for outputs or uses an existing one with the given name") |
| 19 |
> |
parser.add_option("-N", "--number-of-events", dest="nevents_split", default=100000, |
| 20 |
> |
help="Number of events per file when splitting.") |
| 21 |
> |
parser.add_option("-P", "--philipp-love-progress-bars", dest="philipp_love_progress_bars", default=False, |
| 22 |
> |
help="If you share the love of Philipp...") |
| 23 |
|
|
| 24 |
|
(opts, args) = parser.parse_args(sys.argv) |
| 25 |
+ |
|
| 26 |
+ |
import os,shutil,pickle,subprocess,ROOT,re |
| 27 |
+ |
ROOT.gROOT.SetBatch(True) |
| 28 |
+ |
from myutils import BetterConfigParser, Sample, ParseInfo |
| 29 |
+ |
import getpass |
| 30 |
+ |
|
| 31 |
|
if opts.tag == "": |
| 32 |
|
print "Please provide tag to run the analysis with, example '-T 8TeV' uses config8TeV and pathConfig8TeV to run the analysis." |
| 33 |
|
sys.exit(123) |
| 36 |
|
print "Please provide a task.\n-J prep:\tpreparation of Trees\n-J sys:\t\twrite regression and systematics\n-J eval:\tcreate MVA output\n-J plot:\tproduce Plots\n-J dc:\t\twrite workspaces and datacards" |
| 37 |
|
sys.exit(123) |
| 38 |
|
|
| 39 |
+ |
|
| 40 |
+ |
en = opts.tag |
| 41 |
+ |
|
| 42 |
|
#create the list with the samples to run over |
| 43 |
|
samplesList=opts.samples.split(",") |
| 44 |
|
|
| 45 |
< |
en = opts.tag |
| 46 |
< |
configs = ['config%s'%(en),'pathConfig%s'%(en)] |
| 47 |
< |
|
| 45 |
> |
timestamp = time.asctime().replace(' ','_').replace(':','-') |
| 46 |
> |
|
| 47 |
> |
configs = ['%sconfig/general'%(en),'%sconfig/paths'%(en),'%sconfig/plots'%(en),'%sconfig/training'%(en),'%sconfig/datacards'%(en),'%sconfig/cuts'%(en)] |
| 48 |
> |
|
| 49 |
> |
pathconfig = BetterConfigParser() |
| 50 |
> |
pathconfig.read('%sconfig/paths'%(en)) |
| 51 |
> |
|
| 52 |
> |
if not opts.ftag == '': |
| 53 |
> |
tagDir = pathconfig.get('Directories','tagDir') |
| 54 |
> |
DirStruct={'tagDir':tagDir,'ftagdir':'%s/%s/'%(tagDir,opts.ftag),'logpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Logs'),'plotpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Plots'),'limitpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Limits'),'confpath':'%s/%s/%s/'%(tagDir,opts.ftag,'config') } |
| 55 |
> |
|
| 56 |
> |
for keys in ['tagDir','ftagdir','logpath','plotpath','limitpath','confpath']: |
| 57 |
> |
try: |
| 58 |
> |
os.stat(DirStruct[keys]) |
| 59 |
> |
except: |
| 60 |
> |
os.mkdir(DirStruct[keys]) |
| 61 |
> |
|
| 62 |
> |
pathfile = open('%sconfig/paths'%(en)) |
| 63 |
> |
buffer = pathfile.readlines() |
| 64 |
> |
pathfile.close() |
| 65 |
> |
os.rename('%sconfig/paths'%(en),'%sconfig/paths.bkp'%(en)) |
| 66 |
> |
pathfile = open('%sconfig/paths'%(en),'w') |
| 67 |
> |
for line in buffer: |
| 68 |
> |
if line.startswith('plotpath'): |
| 69 |
> |
line = 'plotpath: %s\n'%DirStruct['plotpath'] |
| 70 |
> |
elif line.startswith('logpath'): |
| 71 |
> |
line = 'logpath: %s\n'%DirStruct['logpath'] |
| 72 |
> |
elif line.startswith('limits'): |
| 73 |
> |
line = 'limits: %s\n'%DirStruct['limitpath'] |
| 74 |
> |
pathfile.write(line) |
| 75 |
> |
pathfile.close() |
| 76 |
> |
|
| 77 |
> |
#copy config files |
| 78 |
> |
for item in configs: |
| 79 |
> |
shutil.copyfile(item,'%s/%s/%s'%(tagDir,opts.ftag,item.strip(en))) |
| 80 |
> |
|
| 81 |
> |
|
| 82 |
|
print configs |
| 83 |
|
config = BetterConfigParser() |
| 84 |
|
config.read(configs) |
| 85 |
+ |
|
| 86 |
|
btagLibrary = config.get('BTagReshaping','library') |
| 87 |
|
submitDir = os.getcwd() |
| 88 |
|
os.chdir(os.path.dirname(btagLibrary)) |
| 99 |
|
print 'Exit' |
| 100 |
|
sys.exit(-1) |
| 101 |
|
|
| 102 |
< |
repDict = {'en':en,'logpath':logPath,'job':'','task':opts.task} |
| 102 |
> |
repDict = {'en':en,'logpath':logPath,'job':'','task':opts.task,'queue': 'all.q','timestamp':timestamp,'additional':'','job_id':''} |
| 103 |
|
def submit(job,repDict): |
| 104 |
|
repDict['job'] = job |
| 105 |
< |
command = 'qsub -V -cwd -q all.q -N %(job)s_%(en)s%(task)s -o %(logpath)s/%(job)s_%(en)s_%(task)s.out -e %(logpath)s/%(job)s_%(en)s_%(task)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task |
| 105 |
> |
command = 'qsub -V -cwd -q %(queue)s -l h_vmem=6G -N %(job)s_%(en)s%(task)s -o %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.out -e %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task + ' ' + repDict['job_id'] + ' ' + repDict['additional'] |
| 106 |
|
print command |
| 107 |
|
subprocess.call([command], shell=True) |
| 108 |
|
|
| 109 |
+ |
if opts.task == 'train': |
| 110 |
+ |
train_list = (config.get('MVALists','List_for_submitscript')).split(',') |
| 111 |
+ |
print train_list |
| 112 |
+ |
for item in train_list: |
| 113 |
+ |
submit(item,repDict) |
| 114 |
+ |
|
| 115 |
+ |
|
| 116 |
|
if opts.task == 'dc': |
| 117 |
< |
DC_vars = config.items('Limit') |
| 117 |
> |
#DC_vars = config.items('Limit') |
| 118 |
> |
DC_vars= (config.get('LimitGeneral','List')).split(',') |
| 119 |
> |
print DC_vars |
| 120 |
> |
|
| 121 |
|
if opts.task == 'plot': |
| 122 |
< |
Plot_vars= config.items('Plot') |
| 122 |
> |
Plot_vars= (config.get('Plot_general','List')).split(',') |
| 123 |
|
|
| 124 |
|
if not opts.task == 'prep': |
| 125 |
|
path = config.get("Directories","samplepath") |
| 126 |
< |
infofile = open(path+'/env/samples.info','r') |
| 127 |
< |
info = pickle.load(infofile) |
| 69 |
< |
infofile.close() |
| 70 |
< |
|
| 126 |
> |
samplesinfo = config.get("Directories","samplesinfo") |
| 127 |
> |
info = ParseInfo(samplesinfo,path) |
| 128 |
|
|
| 129 |
|
if opts.task == 'plot': |
| 130 |
+ |
repDict['queue'] = 'all.q' |
| 131 |
|
for item in Plot_vars: |
| 132 |
< |
if 'ZH%s'%opts.mass in item[0]: |
| 133 |
< |
submit(item[0],repDict) |
| 76 |
< |
elif opts.mass == '' and 'ZH' in item[0]: |
| 77 |
< |
submit(item[0],repDict) |
| 132 |
> |
submit(item,repDict) |
| 133 |
> |
|
| 134 |
|
|
| 135 |
|
elif opts.task == 'dc': |
| 136 |
+ |
repDict['queue'] = 'all.q' |
| 137 |
|
for item in DC_vars: |
| 138 |
< |
if 'ZH%s'%opts.mass in item[0] and opts.tag in item[0]: |
| 139 |
< |
submit(item[0],repDict) |
| 140 |
< |
elif 'ZH' in item[0] and opts.tag in item[0] and opts.mass == '*': |
| 141 |
< |
submit(item[0],repDict) |
| 138 |
> |
if 'ZH%s'%opts.mass in item: |
| 139 |
> |
submit(item,repDict) |
| 140 |
> |
elif 'ZH' in item and opts.mass == 'all': |
| 141 |
> |
submit(item,repDict) |
| 142 |
> |
|
| 143 |
|
elif opts.task == 'prep': |
| 144 |
|
submit('prepare',repDict) |
| 145 |
|
|
| 146 |
< |
elif opts.task == 'eval' or opts.task == 'sys': |
| 146 |
> |
elif opts.task == 'sys' or opts.task == 'syseval': |
| 147 |
> |
path = config.get("Directories","SYSin") |
| 148 |
> |
samplesinfo = config.get("Directories","samplesinfo") |
| 149 |
> |
info = ParseInfo(samplesinfo,path) |
| 150 |
|
if ( opts.samples == ""): |
| 151 |
|
for job in info: |
| 152 |
+ |
if (job.subsample): continue |
| 153 |
|
submit(job.name,repDict) |
| 154 |
|
else: |
| 155 |
|
for sample in samplesList: |
| 156 |
|
submit(sample,repDict) |
| 157 |
< |
|
| 157 |
> |
|
| 158 |
> |
elif opts.task == 'eval': |
| 159 |
> |
path = config.get("Directories","MVAin") |
| 160 |
> |
samplesinfo = config.get("Directories","samplesinfo") |
| 161 |
> |
info = ParseInfo(samplesinfo,path) |
| 162 |
> |
if ( opts.samples == ""): |
| 163 |
> |
for job in info: |
| 164 |
> |
if (job.subsample): continue |
| 165 |
> |
submit(job.name,repDict) |
| 166 |
> |
else: |
| 167 |
> |
for sample in samplesList: |
| 168 |
> |
submit(sample,repDict) |
| 169 |
> |
|
| 170 |
> |
|
| 171 |
> |
elif( opts.task == 'split' ): |
| 172 |
> |
path = config.get("Directories","SPLITin") |
| 173 |
> |
samplesinfo = config.get("Directories","samplesinfo") |
| 174 |
> |
repDict['additional']=opts.nevents_split |
| 175 |
> |
info = ParseInfo(samplesinfo,path) |
| 176 |
> |
if ( opts.samples == ""): |
| 177 |
> |
for job in info: |
| 178 |
> |
if (job.subsample): continue |
| 179 |
> |
submit(job.name,repDict) |
| 180 |
> |
else: |
| 181 |
> |
for sample in samplesList: |
| 182 |
> |
submit(sample,repDict) |
| 183 |
> |
|
| 184 |
> |
#BDT optimisation |
| 185 |
> |
elif opts.task == 'mva_opt': |
| 186 |
> |
total_number_of_steps=1 |
| 187 |
> |
setting = '' |
| 188 |
> |
for par in (config.get('Optimisation','parameters').split(',')): |
| 189 |
> |
scan_par=eval(config.get('Optimisation',par)) |
| 190 |
> |
setting+=par+'='+str(scan_par[0])+':' |
| 191 |
> |
if len(scan_par) > 1 and scan_par[2] != 0: |
| 192 |
> |
total_number_of_steps+=scan_par[2] |
| 193 |
> |
setting=setting[:-1] # eliminate last column at the end of the setting string |
| 194 |
> |
print setting |
| 195 |
> |
repDict['additional']=setting |
| 196 |
> |
repDict['job_id']=config.get('Optimisation','training') |
| 197 |
> |
submit('OPT_main_set',repDict) |
| 198 |
> |
main_setting=setting |
| 199 |
> |
|
| 200 |
> |
#Scanning all the parameters found in the training config in the Optimisation sector |
| 201 |
> |
for par in (config.get('Optimisation','parameters').split(',')): |
| 202 |
> |
scan_par=eval(config.get('Optimisation',par)) |
| 203 |
> |
print par |
| 204 |
> |
if len(scan_par) > 1 and scan_par[2] != 0: |
| 205 |
> |
for step in range(scan_par[2]): |
| 206 |
> |
value = (scan_par[0])+((1+step)*(scan_par[1]-scan_par[0])/scan_par[2]) |
| 207 |
> |
print value |
| 208 |
> |
setting=re.sub(par+'.*?:',par+'='+str(value)+':',main_setting) |
| 209 |
> |
repDict['additional']=setting |
| 210 |
> |
# repDict['job_id']=config.get('Optimisation','training') |
| 211 |
> |
submit('OPT_'+par+str(value),repDict) |
| 212 |
> |
# submit(config.get('Optimisation','training'),repDict) |
| 213 |
> |
print setting |
| 214 |
> |
|
| 215 |
> |
|
| 216 |
> |
os.system('qstat') |
| 217 |
> |
if (opts.philipp_love_progress_bars): |
| 218 |
> |
os.system('./qstat.py') |
