VHbb/python/submitThem.py

#! /usr/bin/env python
from optparse import OptionParser
import sys
import time
import os
import shutil

parser = OptionParser()
parser.add_option("-T", "--tag", dest="tag", default="",
                      help="Tag to run the analysis with, example '8TeV' uses config8TeV and pathConfig8TeV to run the analysis")
parser.add_option("-J", "--task", dest="task", default="",
                      help="Task to be done, i.e. 'dc' for Datacards, 'prep' for preparation of Trees, 'plot' to produce plots or 'eval' to write the MVA output or 'sys' to write regression and systematics (or 'syseval' for both). ")
parser.add_option("-M", "--mass", dest="mass", default="125",
                      help="Mass for DC or Plots, 110...135")
parser.add_option("-S","--samples",dest="samples",default="",
                      help="samples you want to run on")
parser.add_option("-F", "--folderTag", dest="ftag", default="",
                      help="Creats a new folder structure for outputs or uses an existing one with the given name")
(opts, args) = parser.parse_args(sys.argv)

import os,shutil,pickle,subprocess,ROOT
ROOT.gROOT.SetBatch(True)
from myutils import BetterConfigParser, Sample, ParseInfo
import getpass

if opts.tag == "":
        print "Please provide tag to run the analysis with, example '-T 8TeV' uses config8TeV and pathConfig8TeV to run the analysis."
        sys.exit(123)

if opts.task == "":
    print "Please provide a task.\n-J prep:\tpreparation of Trees\n-J sys:\t\twrite regression and systematics\n-J eval:\tcreate MVA output\n-J plot:\tproduce Plots\n-J dc:\t\twrite workspaces and datacards"
    sys.exit(123)


en = opts.tag

#create the list with the samples to run over
samplesList=opts.samples.split(",")

timestamp = time.asctime().replace(' ','_').replace(':','-')

configs = ['%sconfig/general'%(en),'%sconfig/paths'%(en),'%sconfig/plots'%(en),'%sconfig/training'%(en),'%sconfig/datacards'%(en),'%sconfig/cuts'%(en)]

pathconfig = BetterConfigParser()
pathconfig.read('%sconfig/paths'%(en))

if not opts.ftag == '':
    tagDir = pathconfig.get('Directories','tagDir')
    DirStruct={'tagDir':tagDir,'ftagdir':'%s/%s/'%(tagDir,opts.ftag),'logpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Logs'),'plotpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Plots'),'limitpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Limits'),'confpath':'%s/%s/%s/'%(tagDir,opts.ftag,'config') }

    for keys in ['tagDir','ftagdir','logpath','plotpath','limitpath','confpath']:
        try:
            os.stat(DirStruct[keys])
        except:
            os.mkdir(DirStruct[keys])

    pathfile = open('%sconfig/paths'%(en))
    buffer = pathfile.readlines()
    pathfile.close()
    os.rename('%sconfig/paths'%(en),'%sconfig/paths.bkp'%(en))
    pathfile = open('%sconfig/paths'%(en),'w')
    for line in buffer:
        if line.startswith('plotpath'):
            line = 'plotpath: %s\n'%DirStruct['plotpath']
        elif line.startswith('logpath'):
            line = 'logpath: %s\n'%DirStruct['logpath']
        elif line.startswith('limits'):
            line = 'limits: %s\n'%DirStruct['limitpath']
        pathfile.write(line)
    pathfile.close()

    #copy config files
    for item in configs:
        shutil.copyfile(item,'%s/%s/%s'%(tagDir,opts.ftag,item.strip(en)))


print configs
config = BetterConfigParser()
config.read(configs)

btagLibrary = config.get('BTagReshaping','library')
submitDir = os.getcwd()
os.chdir(os.path.dirname(btagLibrary))
if not os.path.exists(btagLibrary):
    ROOT.gROOT.LoadMacro('%s+'%btagLibrary.replace('_h.so','.h')) 
shutil.copyfile(os.path.basename(btagLibrary),'/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)))
shutil.copyfile('/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)),btagLibrary)
os.chdir(submitDir)
logPath = config.get("Directories","logpath")
#check if the logPath exist. If not exit
if( not os.path.isdir(logPath) ):
        print 'ERROR: ' + logPath + ': dir not found.'
        print 'ERROR: Create it before submitting '
        print 'Exit'
        sys.exit(-1)

repDict = {'en':en,'logpath':logPath,'job':'','task':opts.task,'queue': 'all.q','timestamp':timestamp}
def submit(job,repDict):
        repDict['job'] = job
        command = 'qsub -V -cwd -q %(queue)s -l h_vmem=6G -N %(job)s_%(en)s%(task)s -o %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.out -e %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task
        print command
        subprocess.call([command], shell=True)

if opts.task == 'dc':
    #DC_vars = config.items('Limit')
    DC_vars= (config.get('LimitGeneral','List')).split(',')
    print DC_vars

if opts.task == 'plot':
    Plot_vars= (config.get('Plot_general','List')).split(',')

if not opts.task == 'prep':
    path = config.get("Directories","samplepath")
    samplesinfo = config.get("Directories","samplesinfo")
    info = ParseInfo(samplesinfo,path)

if opts.task == 'plot': 
    repDict['queue'] = 'all.q'
    for item in Plot_vars:
        submit(item,repDict)

elif opts.task == 'dc':
    repDict['queue'] = 'all.q'
    for item in DC_vars:
        if 'ZH%s'%opts.mass in item:
            submit(item,repDict) 
        elif 'ZH' in item and opts.mass == 'all':
            submit(item,repDict)
            
elif opts.task == 'prep':
    submit('prepare',repDict)

elif opts.task == 'eval' or opts.task == 'sys' or opts.task == 'syseval':
    if ( opts.samples == ""):
        for job in info:
            submit(job.name,repDict)
    else:
        for sample in samplesList:
            submit(sample,repDict)

os.system('qstat') 
os.system('./qstat.py') 
Revision:	1.28
Committed:	Fri Feb 8 14:36:00 2013 UTC (12 years, 3 months ago) by peller
Content type:	text/x-python
Branch:	MAIN
Changes since 1.27:	+48 -5 lines
Log Message:	added tag directory option to submit them
#	User	Rev	Content
1	nmohr	1.1	#! /usr/bin/env python
2	nmohr	1.6	from optparse import OptionParser
3	bortigno	1.26	import sys
4	peller	1.28	import time
5			import os
6			import shutil
7	nmohr	1.7
8	nmohr	1.6	parser = OptionParser()
9			parser.add_option("-T", "--tag", dest="tag", default="",
10			help="Tag to run the analysis with, example '8TeV' uses config8TeV and pathConfig8TeV to run the analysis")
11	peller	1.9	parser.add_option("-J", "--task", dest="task", default="",
12	nmohr	1.19	help="Task to be done, i.e. 'dc' for Datacards, 'prep' for preparation of Trees, 'plot' to produce plots or 'eval' to write the MVA output or 'sys' to write regression and systematics (or 'syseval' for both). ")
13	peller	1.9	parser.add_option("-M", "--mass", dest="mass", default="125",
14	bortigno	1.10	help="Mass for DC or Plots, 110...135")
15			parser.add_option("-S","--samples",dest="samples",default="",
16			help="samples you want to run on")
17	peller	1.28	parser.add_option("-F", "--folderTag", dest="ftag", default="",
18			help="Creats a new folder structure for outputs or uses an existing one with the given name")
19	bortigno	1.26	(opts, args) = parser.parse_args(sys.argv)
20
21			import os,shutil,pickle,subprocess,ROOT
22	nmohr	1.27	ROOT.gROOT.SetBatch(True)
23			from myutils import BetterConfigParser, Sample, ParseInfo
24	bortigno	1.26	import getpass
25	nmohr	1.6
26			if opts.tag == "":
27			print "Please provide tag to run the analysis with, example '-T 8TeV' uses config8TeV and pathConfig8TeV to run the analysis."
28			sys.exit(123)
29	peller	1.9
30			if opts.task == "":
31			print "Please provide a task.\n-J prep:\tpreparation of Trees\n-J sys:\t\twrite regression and systematics\n-J eval:\tcreate MVA output\n-J plot:\tproduce Plots\n-J dc:\t\twrite workspaces and datacards"
32			sys.exit(123)
33
34	peller	1.28
35			en = opts.tag
36
37	bortigno	1.11	#create the list with the samples to run over
38			samplesList=opts.samples.split(",")
39	bortigno	1.10
40	peller	1.28	timestamp = time.asctime().replace(' ','_').replace(':','-')
41
42	nmohr	1.25	configs = ['%sconfig/general'%(en),'%sconfig/paths'%(en),'%sconfig/plots'%(en),'%sconfig/training'%(en),'%sconfig/datacards'%(en),'%sconfig/cuts'%(en)]
43	peller	1.28
44			pathconfig = BetterConfigParser()
45			pathconfig.read('%sconfig/paths'%(en))
46
47			if not opts.ftag == '':
48			tagDir = pathconfig.get('Directories','tagDir')
49			DirStruct={'tagDir':tagDir,'ftagdir':'%s/%s/'%(tagDir,opts.ftag),'logpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Logs'),'plotpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Plots'),'limitpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Limits'),'confpath':'%s/%s/%s/'%(tagDir,opts.ftag,'config') }
50
51			for keys in ['tagDir','ftagdir','logpath','plotpath','limitpath','confpath']:
52			try:
53			os.stat(DirStruct[keys])
54			except:
55			os.mkdir(DirStruct[keys])
56
57			pathfile = open('%sconfig/paths'%(en))
58			buffer = pathfile.readlines()
59			pathfile.close()
60			os.rename('%sconfig/paths'%(en),'%sconfig/paths.bkp'%(en))
61			pathfile = open('%sconfig/paths'%(en),'w')
62			for line in buffer:
63			if line.startswith('plotpath'):
64			line = 'plotpath: %s\n'%DirStruct['plotpath']
65			elif line.startswith('logpath'):
66			line = 'logpath: %s\n'%DirStruct['logpath']
67			elif line.startswith('limits'):
68			line = 'limits: %s\n'%DirStruct['limitpath']
69			pathfile.write(line)
70			pathfile.close()
71
72			#copy config files
73			for item in configs:
74			shutil.copyfile(item,'%s/%s/%s'%(tagDir,opts.ftag,item.strip(en)))
75
76
77	nmohr	1.6	print configs
78			config = BetterConfigParser()
79			config.read(configs)
80	peller	1.28
81	nmohr	1.7	btagLibrary = config.get('BTagReshaping','library')
82			submitDir = os.getcwd()
83			os.chdir(os.path.dirname(btagLibrary))
84			if not os.path.exists(btagLibrary):
85			ROOT.gROOT.LoadMacro('%s+'%btagLibrary.replace('_h.so','.h'))
86			shutil.copyfile(os.path.basename(btagLibrary),'/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)))
87			shutil.copyfile('/scratch/%s/%s'%(getpass.getuser(),os.path.basename(btagLibrary)),btagLibrary)
88			os.chdir(submitDir)
89	nmohr	1.6	logPath = config.get("Directories","logpath")
90	bortigno	1.14	#check if the logPath exist. If not exit
91			if( not os.path.isdir(logPath) ):
92			print 'ERROR: ' + logPath + ': dir not found.'
93			print 'ERROR: Create it before submitting '
94			print 'Exit'
95			sys.exit(-1)
96
97	peller	1.28	repDict = {'en':en,'logpath':logPath,'job':'','task':opts.task,'queue': 'all.q','timestamp':timestamp}
98	nmohr	1.6	def submit(job,repDict):
99			repDict['job'] = job
100	peller	1.28	command = 'qsub -V -cwd -q %(queue)s -l h_vmem=6G -N %(job)s_%(en)s%(task)s -o %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.out -e %(logpath)s/%(timestamp)s_%(job)s_%(en)s_%(task)s.err runAll.sh %(job)s %(en)s ' %(repDict) + opts.task
101	nmohr	1.1	print command
102			subprocess.call([command], shell=True)
103
104	peller	1.9	if opts.task == 'dc':
105	peller	1.18	#DC_vars = config.items('Limit')
106			DC_vars= (config.get('LimitGeneral','List')).split(',')
107			print DC_vars
108
109	peller	1.9	if opts.task == 'plot':
110	peller	1.15	Plot_vars= (config.get('Plot_general','List')).split(',')
111	peller	1.8
112	peller	1.12	if not opts.task == 'prep':
113			path = config.get("Directories","samplepath")
114	nmohr	1.21	samplesinfo = config.get("Directories","samplesinfo")
115	nmohr	1.27	info = ParseInfo(samplesinfo,path)
116	peller	1.8
117	peller	1.9	if opts.task == 'plot':
118	peller	1.23	repDict['queue'] = 'all.q'
119	peller	1.9	for item in Plot_vars:
120	peller	1.15	submit(item,repDict)
121	peller	1.9
122			elif opts.task == 'dc':
123	peller	1.23	repDict['queue'] = 'all.q'
124	peller	1.9	for item in DC_vars:
125	peller	1.22	if 'ZH%s'%opts.mass in item:
126	peller	1.18	submit(item,repDict)
127	peller	1.22	elif 'ZH' in item and opts.mass == 'all':
128	peller	1.18	submit(item,repDict)
129	peller	1.17
130	peller	1.9	elif opts.task == 'prep':
131			submit('prepare',repDict)
132
133	nmohr	1.19	elif opts.task == 'eval' or opts.task == 'sys' or opts.task == 'syseval':
134	bortigno	1.10	if ( opts.samples == ""):
135			for job in info:
136			submit(job.name,repDict)
137			else:
138	bortigno	1.11	for sample in samplesList:
139	bortigno	1.10	submit(sample,repDict)
140	peller	1.20
141			os.system('qstat')
142	peller	1.28	os.system('./qstat.py')