| 1 |
buchmann |
1.1 |
#!/usr/bin/python
|
| 2 |
|
|
|
| 3 |
|
|
# System libraries to interact with the shell
|
| 4 |
|
|
import sys
|
| 5 |
|
|
import os
|
| 6 |
|
|
from os import popen
|
| 7 |
|
|
import commands
|
| 8 |
|
|
import time
|
| 9 |
|
|
|
| 10 |
|
|
|
| 11 |
|
|
jobnumbers=[]
|
| 12 |
|
|
totaljobnumber=0
|
| 13 |
|
|
totjobs=0;
|
| 14 |
|
|
|
| 15 |
|
|
def checklist(joblist,currlist):
|
| 16 |
|
|
jobfinished=[]
|
| 17 |
|
|
for i in joblist:
|
| 18 |
|
|
if i in currlist: ## if the job is still active
|
| 19 |
|
|
currlist.remove(int(i))## we don't need this anymore on the list of currently running jobs (we already know it's running)
|
| 20 |
|
|
else:#job is finished
|
| 21 |
|
|
jobfinished.append(int(i))
|
| 22 |
|
|
for job in jobfinished:
|
| 23 |
|
|
joblist.remove(int(job))
|
| 24 |
|
|
print "Jobs left:"+str(len(joblist))+" / "+str(totjobs)
|
| 25 |
|
|
|
| 26 |
|
|
def create_config():
|
| 27 |
|
|
command="../../Plotting/Create_All_Plots.exec 1" #this is the command to give us the configuration file :-)
|
| 28 |
|
|
print "Creating the configuration"
|
| 29 |
|
|
pipe=popen(command)
|
| 30 |
|
|
print "Everything is sent up. Your jobs will be submitted in a minute!"
|
| 31 |
|
|
|
| 32 |
|
|
def lay_out_groundwork():
|
| 33 |
|
|
if os.path.isfile("../last_configuration.C"):
|
| 34 |
|
|
result="fff"
|
| 35 |
|
|
while(result!="" and result!="new"):
|
| 36 |
|
|
result=""
|
| 37 |
|
|
# result=raw_input("Configuration file exists - would you like to use this one or create a new one? if you want to create a new one, please write new, if you want to use the existing one please either hit enter\n new = generate a new file \n [ENTER] = use existing file: \n")
|
| 38 |
|
|
if result=="":
|
| 39 |
|
|
print "You've hit enter! Using the existing version ... ";
|
| 40 |
|
|
else:
|
| 41 |
|
|
print "You've written \"new\" - creating a new file!";
|
| 42 |
|
|
create_config()
|
| 43 |
|
|
else:
|
| 44 |
|
|
create_config()
|
| 45 |
|
|
|
| 46 |
|
|
def do_simple_submit():
|
| 47 |
|
|
print "Going to submit "+str(sys.argv[1])+" jobs : "
|
| 48 |
|
|
fusecommand="hadd -f output/Asymptotic_Limits.root"
|
| 49 |
|
|
path=str(os.path.abspath(os.curdir))
|
| 50 |
|
|
path=path.replace("/","\/");
|
| 51 |
|
|
savearg=(sys.argv[2]).replace("/","\/")
|
| 52 |
|
|
confcommand="cat Source/ShapeLimWorker.sh_template | sed 's/SETJZBCUT/"+str(sys.argv[1])+"/' | sed 's/SETNJOBS/"+str(sys.argv[1])+"/' | sed 's/THISDIR/"+str(path)+"/' > output/ShapeLimWorker.sh"
|
| 53 |
|
|
print confcommand
|
| 54 |
|
|
commands.getoutput(confcommand)
|
| 55 |
|
|
# sys.exit(-1)
|
| 56 |
|
|
for ijob in range(0,int(sys.argv[1])):
|
| 57 |
|
|
pipe=popen("qsub -q short.q -e /tmp/ -o /tmp/ -N " + "ShaLimA"+str(ijob)+ " " + os.curdir + "/output/ShapeLimWorker.sh"+ " "+str(ijob))
|
| 58 |
|
|
for l in pipe.readlines():
|
| 59 |
|
|
if l.find("Your job")>-1:
|
| 60 |
|
|
thisjobnumber=int(l[l.index('job ')+4:l.index(' (')])
|
| 61 |
|
|
print ": Submitted job "+str(ijob)+" with job number: "+str(thisjobnumber)
|
| 62 |
|
|
jobnumbers.append(thisjobnumber)
|
| 63 |
|
|
fusecommand=fusecommand+" output/DistributedLimits_job"+str(ijob)+"_of_"+str(sys.argv[1])+".root"
|
| 64 |
|
|
counter=0
|
| 65 |
|
|
totjobs=sys.argv[1]
|
| 66 |
|
|
while(len(jobnumbers)>0) :
|
| 67 |
|
|
counter+=1
|
| 68 |
|
|
currlist=[]
|
| 69 |
|
|
pipe=popen("qstat | grep `whoami` | awk '{print $1}'")
|
| 70 |
|
|
for line in pipe.readlines():
|
| 71 |
|
|
currlist.append(int(line))
|
| 72 |
|
|
checklist(jobnumbers,currlist)
|
| 73 |
|
|
time.sleep(60)
|
| 74 |
|
|
print "All jobs have finished - going to merge them, and pick points to do full fledged limit setting"
|
| 75 |
|
|
Asymptotic_Limits
|
| 76 |
|
|
commands.getoutput("python Utilities/DoFullLimits.py "+str(os.path.abspath(os.curdir))+"/output/Asymptotic_Limits.root")
|
| 77 |
|
|
print "All done!"
|
| 78 |
|
|
|
| 79 |
|
|
def do_advanced_submit():
|
| 80 |
|
|
counter=-2;
|
| 81 |
|
|
for argument in sys.argv:
|
| 82 |
|
|
counter+=1
|
| 83 |
|
|
if counter < 1: continue
|
| 84 |
|
|
confcommand="cat basic_sender.py | sed 's/nPEOPLE/"+str(int(len(sys.argv)-2))+"/' | sed 's/nJOBS/"+str(sys.argv[1])+"/' | sed 's/iPEOPLE/"+str(counter)+"/' > distributed_"+argument+".py"
|
| 85 |
|
|
print confcommand
|
| 86 |
|
|
commands.getoutput(confcommand)
|
| 87 |
|
|
|
| 88 |
|
|
def printUsage():
|
| 89 |
|
|
print "Please provide the following argument: (total number of jobs) (abs path to syst file)"
|
| 90 |
|
|
# print "You can also, on top of the number of jobs, provide a list of names (e.g. Adam Barbara Carl Debra ... ) to share the computational load"
|
| 91 |
|
|
|
| 92 |
|
|
if(len(sys.argv) < 3 or sys.argv[1] == "-h"):
|
| 93 |
|
|
printUsage()
|
| 94 |
|
|
sys.exit(-1)
|
| 95 |
|
|
else:
|
| 96 |
|
|
lay_out_groundwork()
|
| 97 |
|
|
print "Will now recompile to account for any possible changes in the configuration (or setup)"
|
| 98 |
|
|
# commands.getoutput("make")
|
| 99 |
|
|
if(len(sys.argv) ==3):
|
| 100 |
|
|
do_simple_submit()
|
| 101 |
|
|
else:
|
| 102 |
|
|
printUsage()
|
| 103 |
|
|
# Note: advanced submitting has not been implemented yet for limit calculations
|
| 104 |
|
|
# do_advanced_submit()
|
| 105 |
|
|
|
