| 41 |
|
|
| 42 |
|
#create the list with the samples to run over |
| 43 |
|
samplesList=opts.samples.split(",") |
| 44 |
– |
|
| 44 |
|
timestamp = time.asctime().replace(' ','_').replace(':','-') |
| 45 |
|
|
| 46 |
< |
configs = ['%sconfig/general'%(en),'%sconfig/paths'%(en),'%sconfig/plots'%(en),'%sconfig/training'%(en),'%sconfig/datacards'%(en),'%sconfig/cuts'%(en)] |
| 48 |
< |
|
| 46 |
> |
# the list of the config is taken from the path config |
| 47 |
|
pathconfig = BetterConfigParser() |
| 48 |
|
pathconfig.read('%sconfig/paths'%(en)) |
| 49 |
+ |
_configs = pathconfig.get('Configuration','List').split(" ") |
| 50 |
+ |
configs = [ '%sconfig/'%(en) + c for c in _configs ] |
| 51 |
|
|
| 52 |
|
if not opts.ftag == '': |
| 53 |
|
tagDir = pathconfig.get('Directories','tagDir') |
| 83 |
|
config = BetterConfigParser() |
| 84 |
|
config.read(configs) |
| 85 |
|
|
| 86 |
< |
btagLibrary = config.get('BTagReshaping','library') |
| 87 |
< |
submitDir = os.getcwd() |
| 88 |
< |
os.chdir(os.path.dirname(btagLibrary)) |
| 89 |
< |
if not os.path.exists(btagLibrary): |
| 90 |
< |
ROOT.gROOT.LoadMacro('%s+'%btagLibrary.replace('_h.so','.h')) |
| 91 |
< |
shutil.copyfile(os.path.basename(btagLibrary),'/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary))) |
| 92 |
< |
shutil.copyfile('/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)),btagLibrary) |
| 93 |
< |
os.chdir(submitDir) |
| 86 |
> |
def dump_config(configs,output_file): |
| 87 |
> |
""" |
| 88 |
> |
Dump all the configs in a output file |
| 89 |
> |
Args: |
| 90 |
> |
output_file: the file where the log will be dumped |
| 91 |
> |
configs: list of files (string) to be dumped |
| 92 |
> |
Returns: |
| 93 |
> |
nothing |
| 94 |
> |
""" |
| 95 |
> |
outf = open(output_file,'w') |
| 96 |
> |
for i in configs: |
| 97 |
> |
try: |
| 98 |
> |
f=open(i,'r') |
| 99 |
> |
outf.write(f.read()) |
| 100 |
> |
except: print '@WARNING: Config' + i + ' not found. It will not be used.' |
| 101 |
> |
|
| 102 |
> |
def compile_macro(config,macro): |
| 103 |
> |
""" |
| 104 |
> |
Creates the library from a macro using CINT compiling it in scratch to avoid |
| 105 |
> |
problems with the linking in the working nodes. |
| 106 |
> |
Args: |
| 107 |
> |
config: configuration file where the macro path is specified |
| 108 |
> |
macro: macro name to be compiled |
| 109 |
> |
Returns: |
| 110 |
> |
nothing |
| 111 |
> |
""" |
| 112 |
> |
submitDir = os.getcwd() |
| 113 |
> |
_macro=macro+'.h' |
| 114 |
> |
library = config.get(macro,'library') |
| 115 |
> |
libDir=os.path.dirname(library) |
| 116 |
> |
os.chdir(libDir) |
| 117 |
> |
if not os.path.exists(library): |
| 118 |
> |
print '@INFO: Compiling ' + _macro |
| 119 |
> |
scratchDir='/scratch/%s/'%(getpass.getuser()) |
| 120 |
> |
shutil.copyfile(libDir+'/'+_macro,'/scratch/%s/%s'%(getpass.getuser(),_macro)) |
| 121 |
> |
os.chdir(scratchDir) |
| 122 |
> |
ROOT.gROOT.ProcessLine('.L %s+'%(scratchDir+_macro)) |
| 123 |
> |
shutil.copyfile('/scratch/%s/%s'%(getpass.getuser(),os.path.basename(library)),library) |
| 124 |
> |
os.chdir(submitDir) |
| 125 |
> |
|
| 126 |
> |
compile_macro(config,'BTagReshaping') |
| 127 |
> |
compile_macro(config,'VHbbNameSpace') |
| 128 |
> |
|
| 129 |
|
logPath = config.get("Directories","logpath") |
| 130 |
+ |
logo = open('%s/data/submit.txt' %config.get('Directories','vhbbpath')).readlines() |
| 131 |
+ |
counter = 0 |
| 132 |
+ |
|
| 133 |
|
#check if the logPath exist. If not exit |
| 134 |
|
if( not os.path.isdir(logPath) ): |
| 135 |
< |
print 'ERROR: ' + logPath + ': dir not found.' |
| 136 |
< |
print 'ERROR: Create it before submitting ' |
| 137 |
< |
print 'Exit' |
| 138 |
< |
sys.exit(-1) |
| 135 |
> |
print '@ERROR : ' + logPath + ': dir not found.' |
| 136 |
> |
print '@ERROR : Create it before submitting ' |
| 137 |
> |
print 'Exit' |
| 138 |
> |
sys.exit(-1) |
| 139 |
> |
|
| 140 |
|
|
| 141 |
|
repDict = {'en':en,'logpath':logPath,'job':'','task':opts.task,'queue': 'all.q','timestamp':timestamp,'additional':'','job_id':''} |
| 142 |
|
def submit(job,repDict): |
| 143 |
< |
repDict['job'] = job |
| 144 |
< |
command = 'qsub -V -cwd -q %(queue)s -l h_vmem=6G -N %(job)s_%(en)s%(task)s -o %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.out -e %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task + ' ' + repDict['job_id'] + ' ' + repDict['additional'] |
| 145 |
< |
print command |
| 146 |
< |
subprocess.call([command], shell=True) |
| 143 |
> |
global counter |
| 144 |
> |
repDict['job'] = job |
| 145 |
> |
nJob = counter % len(logo) |
| 146 |
> |
counter += 1 |
| 147 |
> |
if opts.philipp_love_progress_bars: |
| 148 |
> |
repDict['name'] = '"%s"' %logo[nJob].strip() |
| 149 |
> |
else: |
| 150 |
> |
repDict['name'] = '%(job)s_%(en)s%(task)s' %repDict |
| 151 |
> |
command = 'qsub -V -cwd -q %(queue)s -l h_vmem=6G -N %(name)s -o %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.out -e %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task + ' ' + repDict['job_id'] + ' ' + repDict['additional'] |
| 152 |
> |
print command |
| 153 |
> |
dump_config(configs,"%(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.config" %(repDict)) |
| 154 |
> |
subprocess.call([command], shell=True) |
| 155 |
|
|
| 156 |
|
if opts.task == 'train': |
| 157 |
|
train_list = (config.get('MVALists','List_for_submitscript')).split(',') |
| 178 |
|
for item in Plot_vars: |
| 179 |
|
submit(item,repDict) |
| 180 |
|
|
| 181 |
+ |
if opts.task == 'trainReg': |
| 182 |
+ |
repDict['queue'] = 'all.q' |
| 183 |
+ |
submit('trainReg',repDict) |
| 184 |
+ |
|
| 185 |
|
|
| 186 |
|
elif opts.task == 'dc': |
| 187 |
|
repDict['queue'] = 'short.q' |
| 208 |
|
info = ParseInfo(samplesinfo,path) |
| 209 |
|
if ( opts.samples == ""): |
| 210 |
|
for job in info: |
| 211 |
< |
if (job.subsample): continue #avoid multiple submissions form subsamples |
| 212 |
< |
# TO FIX FOR SPLITTED SAMPLE |
| 211 |
> |
if (job.subsample): |
| 212 |
> |
continue #avoid multiple submissions form subsamples |
| 213 |
> |
# TO FIX FOR SPLITTED SAMPLE |
| 214 |
|
submit(job.name,repDict) |
| 215 |
|
else: |
| 216 |
|
for sample in samplesList: |
| 222 |
|
info = ParseInfo(samplesinfo,path) |
| 223 |
|
if ( opts.samples == ""): |
| 224 |
|
for job in info: |
| 225 |
< |
if (job.subsample): continue #avoid multiple submissions from subsamples |
| 226 |
< |
if(info.checkSplittedSampleName(job.identifier)): # if multiple entries for one name (splitted samples) use the identifier to submit |
| 227 |
< |
print '@INFO: Splitted samples: submit through identifier' |
| 228 |
< |
submit(job.identifier,repDict) |
| 229 |
< |
else: submit(job.name,repDict) |
| 225 |
> |
if (job.subsample): |
| 226 |
> |
continue #avoid multiple submissions from subsamples |
| 227 |
> |
if(info.checkSplittedSampleName(job.identifier)): # if multiple entries for one name (splitted samples) use the identifier to submit |
| 228 |
> |
print '@INFO: Splitted samples: submit through identifier' |
| 229 |
> |
submit(job.identifier,repDict) |
| 230 |
> |
else: submit(job.name,repDict) |
| 231 |
|
else: |
| 232 |
|
for sample in samplesList: |
| 233 |
|
submit(sample,repDict) |
