| 1 |
buchmann |
1.1 |
#!/usr/bin/python
|
| 2 |
|
|
|
| 3 |
|
|
# System libraries to interact with the shell
|
| 4 |
|
|
import sys
|
| 5 |
|
|
import os
|
| 6 |
|
|
from os import popen
|
| 7 |
|
|
import commands
|
| 8 |
|
|
import time
|
| 9 |
|
|
import getpass
|
| 10 |
|
|
from sys import stdout
|
| 11 |
|
|
from optparse import OptionParser
|
| 12 |
|
|
import ftplib
|
| 13 |
|
|
|
| 14 |
|
|
|
| 15 |
|
|
def clean_file(filename):
|
| 16 |
|
|
if os.path.exists(filename):
|
| 17 |
|
|
os.unlink(filename)
|
| 18 |
|
|
|
| 19 |
|
|
def clean():
|
| 20 |
|
|
clean_file("Crab_Jobs/runthis.sh");
|
| 21 |
|
|
clean_file("Crab_Jobs/crabconfig.cfg");
|
| 22 |
|
|
clean_file("Crab_Jobs/Modules.tar");
|
| 23 |
|
|
clean_file("Crab_Jobs/AddFiles.tar");
|
| 24 |
|
|
clean_file("Crab_Jobs/dummy.py");
|
| 25 |
|
|
|
| 26 |
|
|
def get_root_file_name():
|
| 27 |
|
|
filename=""
|
| 28 |
|
|
commandline="ls -ltrh | grep root | grep gcc | awk '{ print $9 }'"
|
| 29 |
|
|
pipe=popen(commandline)
|
| 30 |
|
|
for l in pipe.readlines():
|
| 31 |
|
|
filename=str(l)
|
| 32 |
|
|
return filename.rstrip()
|
| 33 |
|
|
|
| 34 |
|
|
def produce_all_plots(rootpath):
|
| 35 |
|
|
command=" cp ../Limits/PlotScanResults.exec . && ./PlotScanResults.exec "+rootpath
|
| 36 |
|
|
commands.getoutput(command)
|
| 37 |
|
|
command=" rm ../Limits/PlotScanResults.exec"
|
| 38 |
|
|
commands.getoutput(command)
|
| 39 |
|
|
print "All plots have been generated. That's all, now have a look at the results :-)"
|
| 40 |
|
|
|
| 41 |
|
|
def monitor_job(jobname) :
|
| 42 |
|
|
job_done=False
|
| 43 |
|
|
jobdirectory=os.getcwd()+"/Crab_Jobs/"+jobname
|
| 44 |
|
|
print "Job monitoring for "+str(jobname)
|
| 45 |
|
|
confcommandM="~/material/check_status_plus_LIST.sh -r "+jobdirectory
|
| 46 |
|
|
pipe=popen(confcommandM)
|
| 47 |
|
|
for l in pipe.readlines():
|
| 48 |
|
|
stdout.write("\r Monitoring: " + str(l))
|
| 49 |
|
|
stdout.flush()
|
| 50 |
|
|
stdout.write("\r \r\n") #Done with monitoring ... let's clean this up :-)
|
| 51 |
|
|
|
| 52 |
|
|
def ensure_dir(f) :
|
| 53 |
|
|
if not os.path.exists(f):
|
| 54 |
|
|
os.makedirs(f)
|
| 55 |
|
|
|
| 56 |
|
|
def join_directory(path,filelist) :
|
| 57 |
|
|
localpath="/scratch/"+uname+"/"+path
|
| 58 |
|
|
ensure_dir(localpath)
|
| 59 |
|
|
cleanpath=path;
|
| 60 |
|
|
if (cleanpath[len(cleanpath)-1]=="/") : # remove trailing slash
|
| 61 |
|
|
cleanpath=cleanpath[0:len(cleanpath)-2]
|
| 62 |
|
|
filecounter=0
|
| 63 |
|
|
maxsizeperhadd=50
|
| 64 |
|
|
commandlist=[]
|
| 65 |
|
|
command=""
|
| 66 |
|
|
invertedcounter=len(filelist)
|
| 67 |
|
|
for item in filelist:
|
| 68 |
|
|
sys.stdout.write("\r Working on file "+str(filecounter)+" of "+str(len(filelist))+" ("+str(round(100*filecounter/float(len(filelist)),2))+" % done)")
|
| 69 |
|
|
sys.stdout.flush()
|
| 70 |
|
|
copycommand="lcg-cp srm://storage01.lcg.cscs.ch:8443/srm/managerv2?SFN=/pnfs/lcg.cscs.ch/cms/trivcat/store/user/"+uname+"/"+item+" file:////scratch/"+uname+"/"+item
|
| 71 |
|
|
commands.getstatusoutput(copycommand)
|
| 72 |
|
|
command=command+" "+" /scratch/"+uname+"/"+item
|
| 73 |
|
|
filecounter=filecounter+1
|
| 74 |
|
|
invertedcounter=invertedcounter-1
|
| 75 |
|
|
if filecounter == maxsizeperhadd or invertedcounter == 0:
|
| 76 |
|
|
commandlist.append(command)
|
| 77 |
|
|
command=""
|
| 78 |
|
|
filecounter=0
|
| 79 |
|
|
stdout.write("\r \r\n") #clean up
|
| 80 |
|
|
|
| 81 |
|
|
commandcounter=0
|
| 82 |
|
|
fusedfiles=[]
|
| 83 |
|
|
for command in commandlist:
|
| 84 |
|
|
commandcounter=commandcounter+1
|
| 85 |
|
|
fusefile=" /scratch/"+uname+"/"+cleanpath+"/earlystagecommand"+str(commandcounter)+".root"
|
| 86 |
|
|
fusedfiles.append(fusefile)
|
| 87 |
|
|
ccommand="hadd -f "+fusefile+" "+command
|
| 88 |
|
|
print "Fusing "+str(maxsizeperhadd)+" files"
|
| 89 |
|
|
print ccommand
|
| 90 |
|
|
commands.getoutput(ccommand)
|
| 91 |
|
|
command="hadd -f /scratch/"+uname+"/"+cleanpath+".root"
|
| 92 |
|
|
for file in fusedfiles:
|
| 93 |
|
|
command=command+" "+file
|
| 94 |
|
|
commands.getoutput(command)
|
| 95 |
|
|
|
| 96 |
|
|
# deletecommand="rm -r /scratch/"+uname+"/"+path+"/"
|
| 97 |
|
|
print commands.getoutput(deletecommand)
|
| 98 |
|
|
print "All files have been joined; the merged file is in /scratch/"+uname+"/"+cleanpath+".root"
|
| 99 |
|
|
return "/scratch/"+uname+"/"+cleanpath+".root"
|
| 100 |
|
|
|
| 101 |
|
|
|
| 102 |
|
|
def check_directory(path) :
|
| 103 |
|
|
complete_path="/pnfs/lcg.cscs.ch/cms/trivcat/store/user/"+uname+"/"+path+"/"
|
| 104 |
|
|
print "\033[1;34m Going to checkout the subdirectory "+complete_path+" \033[0m "
|
| 105 |
|
|
listoffiles=[]
|
| 106 |
|
|
supposedtobejoined=False;
|
| 107 |
|
|
commandline="lcg-ls -l srm://storage01.lcg.cscs.ch:8443/srm/managerv2?SFN="+complete_path
|
| 108 |
|
|
pipe=popen(commandline)
|
| 109 |
|
|
for l in pipe.readlines():
|
| 110 |
|
|
currentline=l.strip("\n")
|
| 111 |
|
|
if(currentline[0]=="d") :
|
| 112 |
|
|
check_directory(currentline[currentline.find(path):])
|
| 113 |
|
|
else :
|
| 114 |
|
|
if(currentline.count(path) > 0) :
|
| 115 |
|
|
supposedtobejoined=True
|
| 116 |
|
|
listoffiles.append(currentline[currentline.find(path):])
|
| 117 |
|
|
if supposedtobejoined==True:
|
| 118 |
|
|
join_directory(path,listoffiles)
|
| 119 |
|
|
|
| 120 |
|
|
def fuse_job(jobname) :
|
| 121 |
|
|
print "Going to fuse the output."
|
| 122 |
|
|
check_directory("bussolaresults/"+jobname)
|
| 123 |
|
|
|
| 124 |
|
|
def produce_plots(jobname) :
|
| 125 |
|
|
print "Going to produce all plots"
|
| 126 |
|
|
externalcommand= "../Limits/produce_scan_plots.exec /scratch/"+uname+"/bussolaresults/"+jobname+".root "
|
| 127 |
|
|
commands.getoutput(externalcommand)
|
| 128 |
|
|
tarcommand= "tar zfv allplots_"+jobname+".tar.gz ../Limits/Plots/"
|
| 129 |
|
|
commands.getoutput(tarcommand)
|
| 130 |
|
|
|
| 131 |
|
|
def name_not_taken(specialname):
|
| 132 |
|
|
dirList=os.listdir(os.getcwd()+"/Crab_Jobs")
|
| 133 |
|
|
for fname in dirList:
|
| 134 |
|
|
if fname == specialname:
|
| 135 |
|
|
print "There already seems to be a crab job with the same name - please pick a different (less generic?) name"
|
| 136 |
|
|
return 1
|
| 137 |
|
|
return 0
|
| 138 |
|
|
|
| 139 |
|
|
def usage() :
|
| 140 |
|
|
print "Just run this script and it will prompt you for information; if you want to be quicker you can specify the number of jobs (--jobs/-j) and the name of the crab job (--name/-n)"
|
| 141 |
|
|
sys.exit(0)
|
| 142 |
|
|
|
| 143 |
|
|
print "\n\n This script is going to create your crab job and manage it. Since this will require quite a long time to run \n (the script continuously monitors the progress and resubmits any failing jobs) please make sure to run \n it only from a computer that can easily remain online for quite some time (or be prepared to babysit the jobs \n yourself) \n\n"
|
| 144 |
|
|
|
| 145 |
|
|
def getinfo() :
|
| 146 |
|
|
print "Cleaning up first ..."
|
| 147 |
|
|
clean()
|
| 148 |
|
|
njobs=1234567890
|
| 149 |
|
|
name=""
|
| 150 |
|
|
parser = OptionParser()
|
| 151 |
|
|
parser.add_option("-j","--jobs",dest="njobs",help="number of jobs",default=-1)
|
| 152 |
|
|
parser.add_option("-n","--name",dest="name",help="name of crab job",default="")
|
| 153 |
|
|
(options, args) = parser.parse_args()
|
| 154 |
|
|
|
| 155 |
|
|
name=options.name
|
| 156 |
|
|
njobs=int(options.njobs)
|
| 157 |
|
|
|
| 158 |
|
|
if not os.path.exists("Crab_Jobs") :
|
| 159 |
|
|
makecommand="mkdir Crab_Jobs"
|
| 160 |
|
|
commands.getoutput(makecommand)
|
| 161 |
|
|
|
| 162 |
|
|
while(float(njobs)<1 or float(njobs)>5000):
|
| 163 |
|
|
njobs=0
|
| 164 |
|
|
njobs=raw_input("Question 1: How many jobs would you like to split this into? (0<njobs<5000), --jobs=")
|
| 165 |
|
|
if njobs=="": #user hits enter
|
| 166 |
|
|
njobs=-1
|
| 167 |
|
|
print "Accepted number of jobs:"+str(njobs)
|
| 168 |
|
|
while(name=="" or name_not_taken(name)):
|
| 169 |
|
|
name=raw_input("Question 2: Name for this job? [no spaces!] --name=")
|
| 170 |
|
|
print "Accepted job name :"+name
|
| 171 |
|
|
|
| 172 |
|
|
print "We have now established the number of jobs ("+str(njobs)+") as well as the jobs name ("+str(name)+") which will serve as working directory"
|
| 173 |
|
|
confcommand="cat templates/runthis.sh | sed 's/TOTNJOBS/"+str(njobs)+"/' > Crab_Jobs/runthis.sh"
|
| 174 |
|
|
commands.getoutput(confcommand)
|
| 175 |
|
|
confcommand2="cat templates/crabconfig.cfg | sed 's/NJOBS/"+str(njobs)+"/' | sed 's/WORKINGDIR/"+str(name)+"/' > Crab_Jobs/crabconfig.cfg"
|
| 176 |
|
|
commands.getoutput(confcommand2)
|
| 177 |
|
|
|
| 178 |
|
|
print "Generated the run script (runthis.sh) as well as the crab configuration file (crabconfig.cfg); will now attempt to create the crab job"
|
| 179 |
buchmann |
1.2 |
tarcommand2="cp templates/dummy.py Crab_Jobs/ && tar -cvf Crab_Jobs/Modules.tar ../../Plotting/Modules/ && tar -cvf Crab_Jobs/AddFiles.tar ../Limits/TimedLimitCapsule.C && tar -rvf Crab_Jobs/AddFiles.tar ../LimitsFromSystematics/WorkerScript.C && tar -rvf Crab_Jobs/AddFiles.tar ../last_configuration.C && tar -rvf Crab_Jobs/AddFiles.tar ../Systematics/SystematicsWorkerScript.C && cp templates/Fake_Makefile Makefile && tar -rvf Crab_Jobs/AddFiles.tar Makefile && rm Makefile"
|
| 180 |
buchmann |
1.1 |
commands.getoutput(tarcommand2)
|
| 181 |
|
|
|
| 182 |
|
|
rootfilefetchingcommand="bash utils/get_latest_root_version.sh"
|
| 183 |
|
|
commands.getoutput(rootfilefetchingcommand)
|
| 184 |
|
|
|
| 185 |
|
|
latestrootfile=str(get_root_file_name())
|
| 186 |
|
|
tarcommand2a="tar -rf Crab_Jobs/AddFiles.tar "+str(latestrootfile)+""
|
| 187 |
|
|
commands.getoutput(str(tarcommand2a))
|
| 188 |
|
|
|
| 189 |
|
|
crabcommand="cd Crab_Jobs && crab -create -cfg crabconfig.cfg && rm dummy.py && rm dummy.pyc"
|
| 190 |
|
|
pipe=popen(crabcommand)
|
| 191 |
|
|
for l in pipe.readlines():
|
| 192 |
|
|
print l
|
| 193 |
|
|
dirList=os.listdir(os.getcwd()+"/Crab_Jobs")
|
| 194 |
|
|
direxists=0
|
| 195 |
|
|
for fname in dirList:
|
| 196 |
|
|
if fname == name:
|
| 197 |
|
|
direxists=1
|
| 198 |
|
|
if direxists == 0:
|
| 199 |
|
|
print "There seems to have been a problem while trying to generate the crab job; please verify that the crab environment is correctly set up. If you would like to create the crab job manually (and monitor it yourself), you can still do: \n \n cd Crab_Jobs && crab -c crabconfig.cfg -create -submit"
|
| 200 |
|
|
sys.exit(-1)
|
| 201 |
|
|
else:
|
| 202 |
|
|
print "Crab seems to have run correctly. Will now clean up a bit."
|
| 203 |
|
|
confcommand4="cd Crab_Jobs && rm crabconfig.cfg && rm runthis.sh && rm Modules.tar && rm AddFiles.tar"
|
| 204 |
|
|
commands.getoutput(confcommand4)
|
| 205 |
|
|
|
| 206 |
|
|
monitor="doit"
|
| 207 |
|
|
while(monitor!="" and monitor!="me"):
|
| 208 |
|
|
monitor=raw_input("Would you like to submit the jobs manually (e.g. for testing) [me] or would you like this script to submit the jobs and monitor them [enter]?")
|
| 209 |
|
|
if monitor=="me":
|
| 210 |
|
|
print "You want to submit the jobs yourself. Exiting now."
|
| 211 |
|
|
sys.exit(0)
|
| 212 |
|
|
else:
|
| 213 |
|
|
print "This script will submit jobs and monitor them."
|
| 214 |
|
|
confcommand5="cd Crab_Jobs && crab -c "+str(name)+" -submit"
|
| 215 |
|
|
commands.getoutput(confcommand5)
|
| 216 |
|
|
monitor_job(name)
|
| 217 |
|
|
fuse_job(name)
|
| 218 |
|
|
produce_all_plots("/scratch/"+uname+"/bussolaresults/"+name+".root")
|
| 219 |
|
|
print "Done. Your results are in Limits/Plots/ :-)"
|
| 220 |
|
|
|
| 221 |
|
|
uname=getpass.getuser()
|
| 222 |
|
|
getinfo()
|
| 223 |
|
|
|
| 224 |
|
|
|
