VHbb/python/submitThem.py

#! /usr/bin/env python
from optparse import OptionParser
import sys
import time
import os
import shutil

parser = OptionParser()
parser.add_option("-T", "--tag", dest="tag", default="",
                      help="Tag to run the analysis with, example '8TeV' uses config8TeV and pathConfig8TeV to run the analysis")
parser.add_option("-J", "--task", dest="task", default="",
                      help="Task to be done, i.e. 'dc' for Datacards, 'prep' for preparation of Trees, 'plot' to produce plots or 'eval' to write the MVA output or 'sys' to write regression and systematics (or 'syseval' for both). ")
parser.add_option("-M", "--mass", dest="mass", default="125",
                      help="Mass for DC or Plots, 110...135")
parser.add_option("-S","--samples",dest="samples",default="",
                      help="samples you want to run on")
parser.add_option("-F", "--folderTag", dest="ftag", default="",
                      help="Creats a new folder structure for outputs or uses an existing one with the given name")
(opts, args) = parser.parse_args(sys.argv)

import os,shutil,pickle,subprocess,ROOT
ROOT.gROOT.SetBatch(True)
from myutils import BetterConfigParser, Sample, ParseInfo
import getpass

if opts.tag == "":
        print "Please provide tag to run the analysis with, example '-T 8TeV' uses config8TeV and pathConfig8TeV to run the analysis."
        sys.exit(123)

if opts.task == "":
    print "Please provide a task.\n-J prep:\tpreparation of Trees\n-J sys:\t\twrite regression and systematics\n-J eval:\tcreate MVA output\n-J plot:\tproduce Plots\n-J dc:\t\twrite workspaces and datacards"
    sys.exit(123)


en = opts.tag

#create the list with the samples to run over
samplesList=opts.samples.split(",")

timestamp = time.asctime().replace(' ','_').replace(':','-')

configs = ['%sconfig/general'%(en),'%sconfig/paths'%(en),'%sconfig/plots'%(en),'%sconfig/training'%(en),'%sconfig/datacards'%(en),'%sconfig/cuts'%(en)]

pathconfig = BetterConfigParser()
pathconfig.read('%sconfig/paths'%(en))

if not opts.ftag == '':
    tagDir = pathconfig.get('Directories','tagDir')
    DirStruct={'tagDir':tagDir,'ftagdir':'%s/%s/'%(tagDir,opts.ftag),'logpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Logs'),'plotpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Plots'),'limitpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Limits'),'confpath':'%s/%s/%s/'%(tagDir,opts.ftag,'config') }

    for keys in ['tagDir','ftagdir','logpath','plotpath','limitpath','confpath']:
        try:
            os.stat(DirStruct[keys])
        except:
            os.mkdir(DirStruct[keys])

    pathfile = open('%sconfig/paths'%(en))
    buffer = pathfile.readlines()
    pathfile.close()
    os.rename('%sconfig/paths'%(en),'%sconfig/paths.bkp'%(en))
    pathfile = open('%sconfig/paths'%(en),'w')
    for line in buffer:
        if line.startswith('plotpath'):
            line = 'plotpath: %s\n'%DirStruct['plotpath']
        elif line.startswith('logpath'):
            line = 'logpath: %s\n'%DirStruct['logpath']
        elif line.startswith('limits'):
            line = 'limits: %s\n'%DirStruct['limitpath']
        pathfile.write(line)
    pathfile.close()

    #copy config files
    for item in configs:
        shutil.copyfile(item,'%s/%s/%s'%(tagDir,opts.ftag,item.strip(en)))


print configs
config = BetterConfigParser()
config.read(configs)

btagLibrary = config.get('BTagReshaping','library')
submitDir = os.getcwd()
os.chdir(os.path.dirname(btagLibrary))
if not os.path.exists(btagLibrary):
    ROOT.gROOT.LoadMacro('%s+'%btagLibrary.replace('_h.so','.h')) 
shutil.copyfile(os.path.basename(btagLibrary),'/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)))
shutil.copyfile('/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)),btagLibrary)
os.chdir(submitDir)
logPath = config.get("Directories","logpath")
#check if the logPath exist. If not exit
if( not os.path.isdir(logPath) ):
        print 'ERROR: ' + logPath + ': dir not found.'
        print 'ERROR: Create it before submitting '
        print 'Exit'
        sys.exit(-1)

repDict = {'en':en,'logpath':logPath,'job':'','task':opts.task,'queue': 'all.q','timestamp':timestamp}
def submit(job,repDict):
        repDict['job'] = job
        command = 'qsub -V -cwd -q %(queue)s -l h_vmem=6G -N %(job)s_%(en)s%(task)s -o %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.out -e %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task
        print command
        subprocess.call([command], shell=True)

if opts.task == 'train':
    train_list = (config.get('MVALists','List_for_submitscript')).split(',')
    print train_list
    for item in train_list:
        submit(item,repDict)


if opts.task == 'dc':
    #DC_vars = config.items('Limit')
    DC_vars= (config.get('LimitGeneral','List')).split(',')
    print DC_vars

if opts.task == 'plot':
    Plot_vars= (config.get('Plot_general','List')).split(',')

if not opts.task == 'prep':
    path = config.get("Directories","samplepath")
    samplesinfo = config.get("Directories","samplesinfo")
    info = ParseInfo(samplesinfo,path)

if opts.task == 'plot': 
    repDict['queue'] = 'all.q'
    for item in Plot_vars:
        submit(item,repDict)

elif opts.task == 'dc':
    repDict['queue'] = 'all.q'
    for item in DC_vars:
        if 'ZH%s'%opts.mass in item:
            submit(item,repDict) 
        elif 'ZH' in item and opts.mass == 'all':
            submit(item,repDict)
            
elif opts.task == 'prep':
    submit('prepare',repDict)

elif opts.task == 'eval' or opts.task == 'sys' or opts.task == 'syseval':
    if ( opts.samples == ""):
        for job in info:
            submit(job.name,repDict)
    else:
        for sample in samplesList:
            submit(sample,repDict)

os.system('qstat') 
os.system('./qstat.py') 
Revision:	1.29
Committed:	Wed Feb 13 10:06:17 2013 UTC (12 years, 3 months ago) by peller
Content type:	text/x-python
Branch:	MAIN
Changes since 1.28:	+7 -0 lines
Log Message:	added training option
#	User	Rev	Content
1	nmohr	1.1	#! /usr/bin/env python
2	nmohr	1.6	from optparse import OptionParser
3	bortigno	1.26	import sys
4	peller	1.28	import time
5			import os
6			import shutil
7	nmohr	1.7
8	nmohr	1.6	parser = OptionParser()
9			parser.add_option("-T", "--tag", dest="tag", default="",
10			help="Tag to run the analysis with, example '8TeV' uses config8TeV and pathConfig8TeV to run the analysis")
11	peller	1.9	parser.add_option("-J", "--task", dest="task", default="",
12	nmohr	1.19	help="Task to be done, i.e. 'dc' for Datacards, 'prep' for preparation of Trees, 'plot' to produce plots or 'eval' to write the MVA output or 'sys' to write regression and systematics (or 'syseval' for both). ")
13	peller	1.9	parser.add_option("-M", "--mass", dest="mass", default="125",
14	bortigno	1.10	help="Mass for DC or Plots, 110...135")
15			parser.add_option("-S","--samples",dest="samples",default="",
16			help="samples you want to run on")
17	peller	1.28	parser.add_option("-F", "--folderTag", dest="ftag", default="",
18			help="Creats a new folder structure for outputs or uses an existing one with the given name")
19	bortigno	1.26	(opts, args) = parser.parse_args(sys.argv)
20
21			import os,shutil,pickle,subprocess,ROOT
22	nmohr	1.27	ROOT.gROOT.SetBatch(True)
23			from myutils import BetterConfigParser, Sample, ParseInfo
24	bortigno	1.26	import getpass
25	nmohr	1.6
26			if opts.tag == "":
27			print "Please provide tag to run the analysis with, example '-T 8TeV' uses config8TeV and pathConfig8TeV to run the analysis."
28			sys.exit(123)
29	peller	1.9
30			if opts.task == "":
31			print "Please provide a task.\n-J prep:\tpreparation of Trees\n-J sys:\t\twrite regression and systematics\n-J eval:\tcreate MVA output\n-J plot:\tproduce Plots\n-J dc:\t\twrite workspaces and datacards"
32			sys.exit(123)
33
34	peller	1.28
35			en = opts.tag
36
37	bortigno	1.11	#create the list with the samples to run over
38			samplesList=opts.samples.split(",")
39	bortigno	1.10
40	peller	1.28	timestamp = time.asctime().replace(' ','_').replace(':','-')
41
42	nmohr	1.25	configs = ['%sconfig/general'%(en),'%sconfig/paths'%(en),'%sconfig/plots'%(en),'%sconfig/training'%(en),'%sconfig/datacards'%(en),'%sconfig/cuts'%(en)]
43	peller	1.28
44			pathconfig = BetterConfigParser()
45			pathconfig.read('%sconfig/paths'%(en))
46
47			if not opts.ftag == '':
48			tagDir = pathconfig.get('Directories','tagDir')
49			DirStruct={'tagDir':tagDir,'ftagdir':'%s/%s/'%(tagDir,opts.ftag),'logpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Logs'),'plotpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Plots'),'limitpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Limits'),'confpath':'%s/%s/%s/'%(tagDir,opts.ftag,'config') }
50
51			for keys in ['tagDir','ftagdir','logpath','plotpath','limitpath','confpath']:
52			try:
53			os.stat(DirStruct[keys])
54			except:
55			os.mkdir(DirStruct[keys])
56
57			pathfile = open('%sconfig/paths'%(en))
58			buffer = pathfile.readlines()
59			pathfile.close()
60			os.rename('%sconfig/paths'%(en),'%sconfig/paths.bkp'%(en))
61			pathfile = open('%sconfig/paths'%(en),'w')
62			for line in buffer:
63			if line.startswith('plotpath'):
64			line = 'plotpath: %s\n'%DirStruct['plotpath']
65			elif line.startswith('logpath'):
66			line = 'logpath: %s\n'%DirStruct['logpath']
67			elif line.startswith('limits'):
68			line = 'limits: %s\n'%DirStruct['limitpath']
69			pathfile.write(line)
70			pathfile.close()
71
72			#copy config files
73			for item in configs:
74			shutil.copyfile(item,'%s/%s/%s'%(tagDir,opts.ftag,item.strip(en)))
75
76
77	nmohr	1.6	print configs
78			config = BetterConfigParser()
79			config.read(configs)
80	peller	1.28
81	nmohr	1.7	btagLibrary = config.get('BTagReshaping','library')
82			submitDir = os.getcwd()
83			os.chdir(os.path.dirname(btagLibrary))
84			if not os.path.exists(btagLibrary):
85			ROOT.gROOT.LoadMacro('%s+'%btagLibrary.replace('_h.so','.h'))
86			shutil.copyfile(os.path.basename(btagLibrary),'/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)))
87			shutil.copyfile('/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)),btagLibrary)
88			os.chdir(submitDir)
89	nmohr	1.6	logPath = config.get("Directories","logpath")
90	bortigno	1.14	#check if the logPath exist. If not exit
91			if( not os.path.isdir(logPath) ):
92			print 'ERROR: ' + logPath + ': dir not found.'
93			print 'ERROR: Create it before submitting '
94			print 'Exit'
95			sys.exit(-1)
96
97	peller	1.28	repDict = {'en':en,'logpath':logPath,'job':'','task':opts.task,'queue': 'all.q','timestamp':timestamp}
98	nmohr	1.6	def submit(job,repDict):
99			repDict['job'] = job
100	peller	1.28	command = 'qsub -V -cwd -q %(queue)s -l h_vmem=6G -N %(job)s_%(en)s%(task)s -o %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.out -e %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task
101	nmohr	1.1	print command
102			subprocess.call([command], shell=True)
103
104	peller	1.29	if opts.task == 'train':
105			train_list = (config.get('MVALists','List_for_submitscript')).split(',')
106			print train_list
107			for item in train_list:
108			submit(item,repDict)
109
110
111	peller	1.9	if opts.task == 'dc':
112	peller	1.18	#DC_vars = config.items('Limit')
113			DC_vars= (config.get('LimitGeneral','List')).split(',')
114			print DC_vars
115
116	peller	1.9	if opts.task == 'plot':
117	peller	1.15	Plot_vars= (config.get('Plot_general','List')).split(',')
118	peller	1.8
119	peller	1.12	if not opts.task == 'prep':
120			path = config.get("Directories","samplepath")
121	nmohr	1.21	samplesinfo = config.get("Directories","samplesinfo")
122	nmohr	1.27	info = ParseInfo(samplesinfo,path)
123	peller	1.8
124	peller	1.9	if opts.task == 'plot':
125	peller	1.23	repDict['queue'] = 'all.q'
126	peller	1.9	for item in Plot_vars:
127	peller	1.15	submit(item,repDict)
128	peller	1.9
129			elif opts.task == 'dc':
130	peller	1.23	repDict['queue'] = 'all.q'
131	peller	1.9	for item in DC_vars:
132	peller	1.22	if 'ZH%s'%opts.mass in item:
133	peller	1.18	submit(item,repDict)
134	peller	1.22	elif 'ZH' in item and opts.mass == 'all':
135	peller	1.18	submit(item,repDict)
136	peller	1.17
137	peller	1.9	elif opts.task == 'prep':
138			submit('prepare',repDict)
139
140	nmohr	1.19	elif opts.task == 'eval' or opts.task == 'sys' or opts.task == 'syseval':
141	bortigno	1.10	if ( opts.samples == ""):
142			for job in info:
143			submit(job.name,repDict)
144			else:
145	bortigno	1.11	for sample in samplesList:
146	bortigno	1.10	submit(sample,repDict)
147	peller	1.20
148			os.system('qstat')
149	peller	1.28	os.system('./qstat.py')