| 41 |
|
|
| 42 |
|
#create the list with the samples to run over |
| 43 |
|
samplesList=opts.samples.split(",") |
| 44 |
– |
|
| 44 |
|
timestamp = time.asctime().replace(' ','_').replace(':','-') |
| 45 |
|
|
| 46 |
< |
configs = ['%sconfig/general'%(en),'%sconfig/paths'%(en),'%sconfig/plots'%(en),'%sconfig/training'%(en),'%sconfig/datacards'%(en),'%sconfig/cuts'%(en)] |
| 48 |
< |
|
| 46 |
> |
# the list of the config is taken from the path config |
| 47 |
|
pathconfig = BetterConfigParser() |
| 48 |
|
pathconfig.read('%sconfig/paths'%(en)) |
| 49 |
+ |
_configs = pathconfig.get('Configuration','List').split(" ") |
| 50 |
+ |
configs = [ '%sconfig/'%(en) + c for c in _configs ] |
| 51 |
|
|
| 52 |
|
if not opts.ftag == '': |
| 53 |
|
tagDir = pathconfig.get('Directories','tagDir') |
| 83 |
|
config = BetterConfigParser() |
| 84 |
|
config.read(configs) |
| 85 |
|
|
| 86 |
< |
btagLibrary = config.get('BTagReshaping','library') |
| 87 |
< |
submitDir = os.getcwd() |
| 88 |
< |
os.chdir(os.path.dirname(btagLibrary)) |
| 89 |
< |
if not os.path.exists(btagLibrary): |
| 90 |
< |
ROOT.gROOT.LoadMacro('%s+'%btagLibrary.replace('_h.so','.h')) |
| 91 |
< |
shutil.copyfile(os.path.basename(btagLibrary),'/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary))) |
| 92 |
< |
shutil.copyfile('/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)),btagLibrary) |
| 93 |
< |
os.chdir(submitDir) |
| 86 |
> |
|
| 87 |
> |
def compile_macro(config,macro): |
| 88 |
> |
""" |
| 89 |
> |
Creates the library from a macro using CINT compiling it in scratch to avoid |
| 90 |
> |
problems with the linking in the working nodes. |
| 91 |
> |
Args: |
| 92 |
> |
config: configuration file where the macro path is specified |
| 93 |
> |
macro: macro name to be compiled |
| 94 |
> |
Returns: |
| 95 |
> |
nothing |
| 96 |
> |
""" |
| 97 |
> |
submitDir = os.getcwd() |
| 98 |
> |
_macro=macro+'.h' |
| 99 |
> |
library = config.get(macro,'library') |
| 100 |
> |
libDir=os.path.dirname(library) |
| 101 |
> |
os.chdir(libDir) |
| 102 |
> |
if not os.path.exists(library): |
| 103 |
> |
print '@INFO: Compiling ' + _macro |
| 104 |
> |
scratchDir='/scratch/%s/'%(getpass.getuser()) |
| 105 |
> |
shutil.copyfile(libDir+'/'+_macro,'/scratch/%s/%s'%(getpass.getuser(),_macro)) |
| 106 |
> |
os.chdir(scratchDir) |
| 107 |
> |
ROOT.gROOT.ProcessLine('.L %s+'%(scratchDir+_macro)) |
| 108 |
> |
shutil.copyfile('/scratch/%s/%s'%(getpass.getuser(),os.path.basename(library)),library) |
| 109 |
> |
os.chdir(submitDir) |
| 110 |
> |
|
| 111 |
> |
compile_macro(config,'BTagReshaping') |
| 112 |
> |
compile_macro(config,'VHbbNameSpace') |
| 113 |
> |
|
| 114 |
|
logPath = config.get("Directories","logpath") |
| 115 |
+ |
logo = open('%s/data/submit.txt' %config.get('Directories','vhbbpath')).readlines() |
| 116 |
+ |
counter = 0 |
| 117 |
+ |
|
| 118 |
|
#check if the logPath exist. If not exit |
| 119 |
|
if( not os.path.isdir(logPath) ): |
| 120 |
< |
print 'ERROR: ' + logPath + ': dir not found.' |
| 121 |
< |
print 'ERROR: Create it before submitting ' |
| 122 |
< |
print 'Exit' |
| 123 |
< |
sys.exit(-1) |
| 120 |
> |
print '@ERROR : ' + logPath + ': dir not found.' |
| 121 |
> |
print '@ERROR : Create it before submitting ' |
| 122 |
> |
print 'Exit' |
| 123 |
> |
sys.exit(-1) |
| 124 |
> |
|
| 125 |
|
|
| 126 |
|
repDict = {'en':en,'logpath':logPath,'job':'','task':opts.task,'queue': 'all.q','timestamp':timestamp,'additional':'','job_id':''} |
| 127 |
|
def submit(job,repDict): |
| 128 |
< |
repDict['job'] = job |
| 129 |
< |
command = 'qsub -V -cwd -q %(queue)s -l h_vmem=6G -N %(job)s_%(en)s%(task)s -o %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.out -e %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task + ' ' + repDict['job_id'] + ' ' + repDict['additional'] |
| 130 |
< |
print command |
| 131 |
< |
subprocess.call([command], shell=True) |
| 128 |
> |
global counter |
| 129 |
> |
repDict['job'] = job |
| 130 |
> |
nJob = counter % len(logo) |
| 131 |
> |
counter += 1 |
| 132 |
> |
if opts.philipp_love_progress_bars: |
| 133 |
> |
repDict['name'] = '"%s"' %logo[nJob].strip() |
| 134 |
> |
else: |
| 135 |
> |
repDict['name'] = '%(job)s_%(en)s%(task)s' %repDict |
| 136 |
> |
command = 'qsub -V -cwd -q %(queue)s -l h_vmem=6G -N %(name)s -o %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.out -e %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task + ' ' + repDict['job_id'] + ' ' + repDict['additional'] |
| 137 |
> |
print command |
| 138 |
> |
subprocess.call([command], shell=True) |
| 139 |
|
|
| 140 |
|
if opts.task == 'train': |
| 141 |
|
train_list = (config.get('MVALists','List_for_submitscript')).split(',') |
| 162 |
|
for item in Plot_vars: |
| 163 |
|
submit(item,repDict) |
| 164 |
|
|
| 165 |
+ |
if opts.task == 'trainReg': |
| 166 |
+ |
repDict['queue'] = 'all.q' |
| 167 |
+ |
submit('trainReg',repDict) |
| 168 |
+ |
|
| 169 |
|
|
| 170 |
|
elif opts.task == 'dc': |
| 171 |
|
repDict['queue'] = 'short.q' |
| 176 |
|
submit(item,repDict) |
| 177 |
|
|
| 178 |
|
elif opts.task == 'prep': |
| 179 |
< |
submit('prepare',repDict) |
| 179 |
> |
if ( opts.samples == ""): |
| 180 |
> |
path = config.get("Directories","PREPin") |
| 181 |
> |
samplesinfo = config.get("Directories","samplesinfo") |
| 182 |
> |
info = ParseInfo(samplesinfo,path) |
| 183 |
> |
for job in info: |
| 184 |
> |
submit(job.name,repDict) |
| 185 |
|
|
| 186 |
+ |
else: |
| 187 |
+ |
for sample in samplesList: |
| 188 |
+ |
submit(sample,repDict) |
| 189 |
|
elif opts.task == 'sys' or opts.task == 'syseval': |
| 190 |
|
path = config.get("Directories","SYSin") |
| 191 |
|
samplesinfo = config.get("Directories","samplesinfo") |
| 192 |
|
info = ParseInfo(samplesinfo,path) |
| 193 |
|
if ( opts.samples == ""): |
| 194 |
|
for job in info: |
| 195 |
< |
if (job.subsample): continue #avoid multiple submissions form subsamples |
| 196 |
< |
# TO FIX FOR SPLITTED SAMPLE |
| 195 |
> |
if (job.subsample): |
| 196 |
> |
continue #avoid multiple submissions form subsamples |
| 197 |
> |
# TO FIX FOR SPLITTED SAMPLE |
| 198 |
|
submit(job.name,repDict) |
| 199 |
|
else: |
| 200 |
|
for sample in samplesList: |
| 206 |
|
info = ParseInfo(samplesinfo,path) |
| 207 |
|
if ( opts.samples == ""): |
| 208 |
|
for job in info: |
| 209 |
< |
if (job.subsample): continue #avoid multiple submissions from subsamples |
| 210 |
< |
if(info.checkSplittedSampleName(job.identifier)): # if multiple entries for one name (splitted samples) use the identifier to submit |
| 211 |
< |
print '@INFO: Splitted samples: submit through identifier' |
| 212 |
< |
submit(job.identifier,repDict) |
| 213 |
< |
else: submit(job.name,repDict) |
| 209 |
> |
if (job.subsample): |
| 210 |
> |
continue #avoid multiple submissions from subsamples |
| 211 |
> |
if(info.checkSplittedSampleName(job.identifier)): # if multiple entries for one name (splitted samples) use the identifier to submit |
| 212 |
> |
print '@INFO: Splitted samples: submit through identifier' |
| 213 |
> |
submit(job.identifier,repDict) |
| 214 |
> |
else: submit(job.name,repDict) |
| 215 |
|
else: |
| 216 |
|
for sample in samplesList: |
| 217 |
|
submit(sample,repDict) |
