| 6 |
|
import shutil |
| 7 |
|
|
| 8 |
|
parser = OptionParser() |
| 9 |
< |
parser.add_option("-T", "--tag", dest="tag", default="", |
| 9 |
> |
parser.add_option("-T", "--tag", dest="tag", default="8TeV", |
| 10 |
|
help="Tag to run the analysis with, example '8TeV' uses config8TeV and pathConfig8TeV to run the analysis") |
| 11 |
|
parser.add_option("-J", "--task", dest="task", default="", |
| 12 |
|
help="Task to be done, i.e. 'dc' for Datacards, 'prep' for preparation of Trees, 'plot' to produce plots or 'eval' to write the MVA output or 'sys' to write regression and systematics (or 'syseval' for both). ") |
| 13 |
|
parser.add_option("-M", "--mass", dest="mass", default="125", |
| 14 |
< |
help="Mass for DC or Plots, 110...135") |
| 14 |
> |
help="Mass for DC or Plots, 110...135") |
| 15 |
|
parser.add_option("-S","--samples",dest="samples",default="", |
| 16 |
< |
help="samples you want to run on") |
| 16 |
> |
help="samples you want to run on") |
| 17 |
|
parser.add_option("-F", "--folderTag", dest="ftag", default="", |
| 18 |
|
help="Creats a new folder structure for outputs or uses an existing one with the given name") |
| 19 |
|
parser.add_option("-N", "--number-of-events", dest="nevents_split", default=100000, |
| 29 |
|
import getpass |
| 30 |
|
|
| 31 |
|
if opts.tag == "": |
| 32 |
< |
print "Please provide tag to run the analysis with, example '-T 8TeV' uses config8TeV and pathConfig8TeV to run the analysis." |
| 33 |
< |
sys.exit(123) |
| 32 |
> |
print "Please provide tag to run the analysis with, example '-T 8TeV' uses config8TeV and pathConfig8TeV to run the analysis." |
| 33 |
> |
sys.exit(123) |
| 34 |
|
|
| 35 |
|
if opts.task == "": |
| 36 |
|
print "Please provide a task.\n-J prep:\tpreparation of Trees\n-J sys:\t\twrite regression and systematics\n-J eval:\tcreate MVA output\n-J plot:\tproduce Plots\n-J dc:\t\twrite workspaces and datacards" |
| 41 |
|
|
| 42 |
|
#create the list with the samples to run over |
| 43 |
|
samplesList=opts.samples.split(",") |
| 44 |
– |
|
| 44 |
|
timestamp = time.asctime().replace(' ','_').replace(':','-') |
| 45 |
|
|
| 46 |
< |
configs = ['%sconfig/general'%(en),'%sconfig/paths'%(en),'%sconfig/plots'%(en),'%sconfig/training'%(en),'%sconfig/datacards'%(en),'%sconfig/cuts'%(en)] |
| 48 |
< |
|
| 46 |
> |
# the list of the config is taken from the path config |
| 47 |
|
pathconfig = BetterConfigParser() |
| 48 |
|
pathconfig.read('%sconfig/paths'%(en)) |
| 49 |
+ |
_configs = pathconfig.get('Configuration','List').split(" ") |
| 50 |
+ |
configs = [ '%sconfig/'%(en) + c for c in _configs ] |
| 51 |
|
|
| 52 |
|
if not opts.ftag == '': |
| 53 |
|
tagDir = pathconfig.get('Directories','tagDir') |
| 83 |
|
config = BetterConfigParser() |
| 84 |
|
config.read(configs) |
| 85 |
|
|
| 86 |
< |
btagLibrary = config.get('BTagReshaping','library') |
| 87 |
< |
submitDir = os.getcwd() |
| 88 |
< |
os.chdir(os.path.dirname(btagLibrary)) |
| 89 |
< |
if not os.path.exists(btagLibrary): |
| 90 |
< |
ROOT.gROOT.LoadMacro('%s+'%btagLibrary.replace('_h.so','.h')) |
| 91 |
< |
shutil.copyfile(os.path.basename(btagLibrary),'/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary))) |
| 92 |
< |
shutil.copyfile('/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)),btagLibrary) |
| 93 |
< |
os.chdir(submitDir) |
| 86 |
> |
def dump_config(configs,output_file): |
| 87 |
> |
""" |
| 88 |
> |
Dump all the configs in a output file |
| 89 |
> |
Args: |
| 90 |
> |
output_file: the file where the log will be dumped |
| 91 |
> |
configs: list of files (string) to be dumped |
| 92 |
> |
Returns: |
| 93 |
> |
nothing |
| 94 |
> |
""" |
| 95 |
> |
outf = open(output_file,'w') |
| 96 |
> |
for i in configs: |
| 97 |
> |
try: |
| 98 |
> |
f=open(i,'r') |
| 99 |
> |
outf.write(f.read()) |
| 100 |
> |
except: print '@WARNING: Config' + i + ' not found. It will not be used.' |
| 101 |
> |
|
| 102 |
> |
def compile_macro(config,macro): |
| 103 |
> |
""" |
| 104 |
> |
Creates the library from a macro using CINT compiling it in scratch to avoid |
| 105 |
> |
problems with the linking in the working nodes. |
| 106 |
> |
Args: |
| 107 |
> |
config: configuration file where the macro path is specified |
| 108 |
> |
macro: macro name to be compiled |
| 109 |
> |
Returns: |
| 110 |
> |
nothing |
| 111 |
> |
""" |
| 112 |
> |
submitDir = os.getcwd() |
| 113 |
> |
_macro=macro+'.h' |
| 114 |
> |
library = config.get(macro,'library') |
| 115 |
> |
libDir=os.path.dirname(library) |
| 116 |
> |
os.chdir(libDir) |
| 117 |
> |
if not os.path.exists(library): |
| 118 |
> |
print '@INFO: Compiling ' + _macro |
| 119 |
> |
scratchDir='/scratch/%s/'%(getpass.getuser()) |
| 120 |
> |
shutil.copyfile(libDir+'/'+_macro,'/scratch/%s/%s'%(getpass.getuser(),_macro)) |
| 121 |
> |
os.chdir(scratchDir) |
| 122 |
> |
ROOT.gROOT.ProcessLine('.L %s+'%(scratchDir+_macro)) |
| 123 |
> |
shutil.copyfile('/scratch/%s/%s'%(getpass.getuser(),os.path.basename(library)),library) |
| 124 |
> |
os.chdir(submitDir) |
| 125 |
> |
|
| 126 |
> |
compile_macro(config,'BTagReshaping') |
| 127 |
> |
compile_macro(config,'VHbbNameSpace') |
| 128 |
> |
|
| 129 |
|
logPath = config.get("Directories","logpath") |
| 130 |
+ |
logo = open('%s/data/submit.txt' %config.get('Directories','vhbbpath')).readlines() |
| 131 |
+ |
counter = 0 |
| 132 |
+ |
|
| 133 |
|
#check if the logPath exist. If not exit |
| 134 |
|
if( not os.path.isdir(logPath) ): |
| 135 |
< |
print 'ERROR: ' + logPath + ': dir not found.' |
| 136 |
< |
print 'ERROR: Create it before submitting ' |
| 137 |
< |
print 'Exit' |
| 138 |
< |
sys.exit(-1) |
| 139 |
< |
|
| 135 |
> |
print '@ERROR : ' + logPath + ': dir not found.' |
| 136 |
> |
print '@ERROR : Create it before submitting ' |
| 137 |
> |
print 'Exit' |
| 138 |
> |
sys.exit(-1) |
| 139 |
> |
|
| 140 |
|
repDict = {'en':en,'logpath':logPath,'job':'','task':opts.task,'queue': 'all.q','timestamp':timestamp,'additional':'','job_id':''} |
| 141 |
|
def submit(job,repDict): |
| 142 |
< |
repDict['job'] = job |
| 143 |
< |
command = 'qsub -V -cwd -q %(queue)s -l h_vmem=6G -N %(job)s_%(en)s%(task)s -o %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.out -e %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task + ' ' + repDict['job_id'] + ' ' + repDict['additional'] |
| 144 |
< |
print command |
| 145 |
< |
subprocess.call([command], shell=True) |
| 142 |
> |
global counter |
| 143 |
> |
repDict['job'] = job |
| 144 |
> |
nJob = counter % len(logo) |
| 145 |
> |
counter += 1 |
| 146 |
> |
if opts.philipp_love_progress_bars: |
| 147 |
> |
repDict['name'] = '"%s"' %logo[nJob].strip() |
| 148 |
> |
else: |
| 149 |
> |
repDict['name'] = '%(job)s_%(en)s%(task)s' %repDict |
| 150 |
> |
command = 'qsub -V -cwd -q %(queue)s -l h_vmem=6G -N %(name)s -o %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.out -e %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task + ' ' + repDict['job_id'] + ' ' + repDict['additional'] |
| 151 |
> |
print command |
| 152 |
> |
dump_config(configs,"%(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.config" %(repDict)) |
| 153 |
> |
subprocess.call([command], shell=True) |
| 154 |
|
|
| 155 |
|
if opts.task == 'train': |
| 156 |
|
train_list = (config.get('MVALists','List_for_submitscript')).split(',') |
| 177 |
|
for item in Plot_vars: |
| 178 |
|
submit(item,repDict) |
| 179 |
|
|
| 180 |
+ |
if opts.task == 'trainReg': |
| 181 |
+ |
repDict['queue'] = 'all.q' |
| 182 |
+ |
submit('trainReg',repDict) |
| 183 |
+ |
|
| 184 |
|
|
| 185 |
|
elif opts.task == 'dc': |
| 186 |
< |
repDict['queue'] = 'short.q' |
| 186 |
> |
repDict['queue'] = 'all.q' |
| 187 |
|
for item in DC_vars: |
| 188 |
|
if 'ZH%s'%opts.mass in item: |
| 189 |
|
submit(item,repDict) |
| 190 |
< |
elif 'ZH' in item and opts.mass == 'all': |
| 190 |
> |
elif opts.mass == 'all': |
| 191 |
|
submit(item,repDict) |
| 192 |
|
|
| 193 |
|
elif opts.task == 'prep': |
| 194 |
< |
submit('prepare',repDict) |
| 194 |
> |
if ( opts.samples == ""): |
| 195 |
> |
path = config.get("Directories","PREPin") |
| 196 |
> |
samplesinfo = config.get("Directories","samplesinfo") |
| 197 |
> |
info = ParseInfo(samplesinfo,path) |
| 198 |
> |
for job in info: |
| 199 |
> |
submit(job.name,repDict) |
| 200 |
|
|
| 201 |
+ |
else: |
| 202 |
+ |
for sample in samplesList: |
| 203 |
+ |
submit(sample,repDict) |
| 204 |
|
elif opts.task == 'sys' or opts.task == 'syseval': |
| 205 |
|
path = config.get("Directories","SYSin") |
| 206 |
|
samplesinfo = config.get("Directories","samplesinfo") |
| 207 |
|
info = ParseInfo(samplesinfo,path) |
| 208 |
< |
if ( opts.samples == ""): |
| 208 |
> |
if opts.samples == "": |
| 209 |
|
for job in info: |
| 210 |
< |
if (job.subsample): continue #avoid multiple submissions form subsamples |
| 211 |
< |
# TO FIX FOR SPLITTED SAMPLE |
| 210 |
> |
if (job.subsample): |
| 211 |
> |
continue #avoid multiple submissions form subsamples |
| 212 |
> |
# TO FIX FOR SPLITTED SAMPLE |
| 213 |
|
submit(job.name,repDict) |
| 214 |
|
else: |
| 215 |
|
for sample in samplesList: |
| 216 |
|
submit(sample,repDict) |
| 217 |
|
|
| 218 |
|
elif opts.task == 'eval': |
| 219 |
+ |
repDict['queue'] = 'long.q' |
| 220 |
|
path = config.get("Directories","MVAin") |
| 221 |
|
samplesinfo = config.get("Directories","samplesinfo") |
| 222 |
|
info = ParseInfo(samplesinfo,path) |
| 223 |
< |
if ( opts.samples == ""): |
| 223 |
> |
if opts.samples == "": |
| 224 |
|
for job in info: |
| 225 |
< |
if (job.subsample): continue #avoid multiple submissions from subsamples |
| 226 |
< |
if(info.checkSplittedSampleName(job.identifier)): # if multiple entries for one name (splitted samples) use the identifier to submit |
| 227 |
< |
print '@INFO: Splitted samples: submit through identifier' |
| 228 |
< |
submit(job.identifier,repDict) |
| 229 |
< |
else: submit(job.name,repDict) |
| 225 |
> |
if (job.subsample): |
| 226 |
> |
continue #avoid multiple submissions from subsamples |
| 227 |
> |
if(info.checkSplittedSampleName(job.identifier)): # if multiple entries for one name (splitted samples) use the identifier to submit |
| 228 |
> |
print '@INFO: Splitted samples: submit through identifier' |
| 229 |
> |
submit(job.identifier,repDict) |
| 230 |
> |
else: submit(job.name,repDict) |
| 231 |
|
else: |
| 232 |
|
for sample in samplesList: |
| 233 |
+ |
print sample |
| 234 |
|
submit(sample,repDict) |
| 235 |
|
|
| 236 |
|
|
| 249 |
|
|
| 250 |
|
#BDT optimisation |
| 251 |
|
elif opts.task == 'mva_opt': |
| 252 |
< |
total_number_of_steps=1 |
| 253 |
< |
setting = '' |
| 254 |
< |
for par in (config.get('Optimisation','parameters').split(',')): |
| 255 |
< |
scan_par=eval(config.get('Optimisation',par)) |
| 256 |
< |
setting+=par+'='+str(scan_par[0])+':' |
| 257 |
< |
if len(scan_par) > 1 and scan_par[2] != 0: |
| 258 |
< |
total_number_of_steps+=scan_par[2] |
| 259 |
< |
setting=setting[:-1] # eliminate last column at the end of the setting string |
| 260 |
< |
print setting |
| 261 |
< |
repDict['additional']=setting |
| 262 |
< |
repDict['job_id']=config.get('Optimisation','training') |
| 263 |
< |
submit('OPT_main_set',repDict) |
| 264 |
< |
main_setting=setting |
| 265 |
< |
|
| 266 |
< |
#Scanning all the parameters found in the training config in the Optimisation sector |
| 267 |
< |
for par in (config.get('Optimisation','parameters').split(',')): |
| 268 |
< |
scan_par=eval(config.get('Optimisation',par)) |
| 269 |
< |
print par |
| 270 |
< |
if len(scan_par) > 1 and scan_par[2] != 0: |
| 271 |
< |
for step in range(scan_par[2]): |
| 272 |
< |
value = (scan_par[0])+((1+step)*(scan_par[1]-scan_par[0])/scan_par[2]) |
| 273 |
< |
print value |
| 274 |
< |
setting=re.sub(par+'.*?:',par+'='+str(value)+':',main_setting) |
| 275 |
< |
repDict['additional']=setting |
| 276 |
< |
# repDict['job_id']=config.get('Optimisation','training') |
| 277 |
< |
submit('OPT_'+par+str(value),repDict) |
| 278 |
< |
# submit(config.get('Optimisation','training'),repDict) |
| 279 |
< |
print setting |
| 252 |
> |
total_number_of_steps=1 |
| 253 |
> |
setting = '' |
| 254 |
> |
for par in (config.get('Optimisation','parameters').split(',')): |
| 255 |
> |
scan_par=eval(config.get('Optimisation',par)) |
| 256 |
> |
setting+=par+'='+str(scan_par[0])+':' |
| 257 |
> |
if len(scan_par) > 1 and scan_par[2] != 0: |
| 258 |
> |
total_number_of_steps+=scan_par[2] |
| 259 |
> |
setting=setting[:-1] # eliminate last column at the end of the setting string |
| 260 |
> |
print setting |
| 261 |
> |
repDict['additional']=setting |
| 262 |
> |
repDict['job_id']=config.get('Optimisation','training') |
| 263 |
> |
submit('OPT_main_set',repDict) |
| 264 |
> |
main_setting=setting |
| 265 |
> |
|
| 266 |
> |
#Scanning all the parameters found in the training config in the Optimisation sector |
| 267 |
> |
for par in (config.get('Optimisation','parameters').split(',')): |
| 268 |
> |
scan_par=eval(config.get('Optimisation',par)) |
| 269 |
> |
print par |
| 270 |
> |
if len(scan_par) > 1 and scan_par[2] != 0: |
| 271 |
> |
for step in range(scan_par[2]): |
| 272 |
> |
value = (scan_par[0])+((1+step)*(scan_par[1]-scan_par[0])/scan_par[2]) |
| 273 |
> |
print value |
| 274 |
> |
setting=re.sub(par+'.*?:',par+'='+str(value)+':',main_setting) |
| 275 |
> |
repDict['additional']=setting |
| 276 |
> |
# repDict['job_id']=config.get('Optimisation','training') |
| 277 |
> |
submit('OPT_'+par+str(value),repDict) |
| 278 |
> |
# submit(config.get('Optimisation','training'),repDict) |
| 279 |
> |
print setting |
| 280 |
|
|
| 281 |
|
|
| 282 |
|
os.system('qstat') |
| 283 |
|
if (opts.philipp_love_progress_bars): |
| 284 |
< |
os.system('./qstat.py') |
| 284 |
> |
os.system('./qstat.py') |
