DistributedModelCalculations/CBAFviaCRAB/SendCrabJob.sh

#!/bin/bash

function usage {
 echo "You can use this script to send crab jobs for scans. You can either run it directly and answer the two questions, or you can answer them directly using -n NAME -j NJOBS"
 exit 1
}
 
function clean_file {
 echo "Cleaning: $1" 
 if [ -e $1 ]; then
     rm $1
 fi
}

function clean {
 clean_file "Crab_Jobs/runthis.sh"
 clean_file "Crab_Jobs/runthis.sh"
 clean_file "Crab_Jobs/crabconfig.cfg"
 clean_file "Crab_Jobs/Modules.tar"
 clean_file "Crab_Jobs/AddFiles.tar"
 clean_file "Crab_Jobs/dummy.py"
}

function produce_all_plots {
 directory=$1

 ## delete previous executable so everything's up-to-date
 clean_file "PlotScanResults.C"

 cp ../Limits/PlotScanResults.C .
 g++ PlotScanResults.C -o PlotScanResults.exec `root-config --glibs --cflags` -lTreePlayer
 clean_file "PlotScanResults.C"
 echo "Compiled script to produce Going to produce plots now"
 PlotScanResults.exec $directory
 clean_file "LOGerr.txt"
 clean_file "Tex.txt"
 clean_file "LOG.txt"
 echo "All plots have been generated. That's all, now have a look at the results :-)"
}

function mail_all_plots {
        tar -cvf Plots.tar $1
        echo "About to send all plots"
####### DO NOT UPLOAD THE SCRIPT IN THIS LOCATION OR MARCO WILL GET *VERY,VERY,VERY MAD* AT YOU
        /shome/buchmann/material/send_mail_with_attachment.sh $1 Plots.tar
        rm Plots.tar
}

 name=""
 njobs=1234567890
 while getopts "j:n:h" OPTION
 do
   case $OPTION in
     j)
       njobs=$OPTARG
       echo "Have received option for number of jobs. njobs=$njobs set".
       ;;
     n)
       name=$OPTARG
       echo "Have received option for name. name=$name set".
       ;;
     h)
       usage
       exit
       ;;
    esac
 done
        
 while [ $njobs -lt 1 ] || [ $njobs -gt 5000 ];
 do
    njobs=0
    echo "Question 1: How many jobs would you like to split this into? (0<njobs<5000), (-j) njobs="
    read njobs
 done
 echo "Accepted number of jobs: $njobs"
 
 if [[ "$name" == "" ]]; then 
    echo ""
    else
    if [ -d "Crab_Jobs/$name" ]; then
     echo "The directory identifier you provided ($name) is taken. Please try again."
     name=""
    else 
     echo "The directory identifier you provided ($name) is ok."
    fi
fi

 if [[ "$name" == "" ]]; then
    echo "Question 2: Name for this job? [no spaces!] (-n) name="
    read name
 fi

 if [ -d "Crab_Jobs/$name" ]; then
     echo "The directory identifier you provided ($name) is also taken. Please start again."
     exit -1
 fi 


 echo "Accepted job name : $name"

 echo "We have now established the number of jobs (${njobs}) as well as the jobs name (${name}) which will serve as working directory"
 mkdir -p Crab_Jobs

 eval "cat templates/runthis.sh | sed 's/TOTNJOBS/${njobs}/' > Crab_Jobs/runthis.sh"
 eval "cat templates/crabconfig.cfg | sed 's/NJOBS/${njobs}/' | sed 's/WORKINGDIR/${name}/' > Crab_Jobs/crabconfig.cfg"

 echo "Have generated the run script (runthis.sh) as well as the crab configuration file (crabconfig.cfg); will now attempt to create the crab job"
 
 cp templates/dummy.py Crab_Jobs/
 cp templates/Fake_Makefile Makefile 

 tar -cvf Crab_Jobs/Modules.tar ../../Plotting/Modules/ >/dev/null 2>/dev/null
 tar -cvf Crab_Jobs/AddFiles.tar ../Limits/TimedLimitCapsule.C >/dev/null 2>/dev/null
 tar -rvf Crab_Jobs/AddFiles.tar ../LimitsFromSystematics/WorkerScript.C >/dev/null 2>/dev/null
 tar -rvf Crab_Jobs/AddFiles.tar ../last_configuration.C >/dev/null 2>/dev/null
 tar -rvf Crab_Jobs/AddFiles.tar ../Systematics/SystematicsWorkerScript.C >/dev/null 2>/dev/null 
 tar -rvf Crab_Jobs/AddFiles.tar Makefile >/dev/null 2>/dev/null

 bash utils/get_latest_root_version.sh
 rootfilename=`ls | grep root | grep gcc`
 echo "Attaching root file $rootfilename to tarball ..." 
 tar -rvf Crab_Jobs/AddFiles.tar $rootfilename >/dev/null 2>/dev/null

 cd Crab_Jobs && crab -create -cfg crabconfig.cfg
 cd ../
 rm Makefile

 if [[ -e Crab_Jobs/$name ]]; then 
    echo "Crab seems to have run correctly. Will now clean up a bit."
    rm Crab_Jobs/AddFiles.tar
    rm Crab_Jobs/Modules.tar
    rm Crab_Jobs/dummy.py
    if [[ -f Crab_Jobs/dummy.pyc ]]; then 
        rm Crab_Jobs/dummy.pyc
    fi
    rm Crab_Jobs/crabconfig.cfg
    rm Crab_Jobs/runthis.sh
 else
   echo "There seems to have been a problem while trying to generate the crab job (can't find any Crab_Jobs/$name directory); please verify that the crab environment is correctly set up. If you would like to create the crab job manually (and monitor it yourself), you can still do: \n \n cd Crab_Jobs && crab -c crabconfig.cfg -create -submit"
    exit -1
 fi

 echo "Would you like to submit the jobs manually (e.g. for testing) [me] or would you like this script to submit the jobs and monitor them [anything else]?"
 read answer
 if [[ "$answer" == "me" ]]; then
         echo "Command you might need:"
         echo "cd Crab_Jobs && crab -c ${name} -submit"
         echo "You're on your own now. Goodbye!"
         exit 0
 fi

 echo "You chose to have this handled automatically. Please sit back, submitting jobs now!"
 cd Crab_Jobs && crab -c ${name} -submit && cd ..
 echo "Ok, everything has been submitted. Now let's check on the status ... (this will take a long time)"
### THIS SCRIPT REMAINS ON THE T3!!! DO NOT UPLOAD IT!
/shome/buchmann/material/check_status_plus_LIST.sh -r `pwd`/Crab_Jobs/${name}

 echo "Now going to fuse everything related to this job: ${name}"
 python utils/fusejobs.py -d ${name}
 rm -r Plots/
 source /swshare/ROOT/thisroot.sh

 rm -r Plots_${name}/
 rm -r Plots/
 produce_all_plots "/scratch/`whoami`/bussolaresults/${name}.root"
 mv Plots/ Plots_${name}/
 mail_all_plots Plots_${name}
 echo "Done. Your results are in Plots_${name}/ :-)"
Revision:	1.2
Committed:	Thu Jan 19 14:14:36 2012 UTC (13 years, 3 months ago) by buchmann
Content type:	application/x-sh
Branch:	MAIN
CVS Tags:	cbaf_4_98ifb_paper, cbaf_4p7ifb, HEAD
Changes since 1.1:	+15 -7 lines
Log Message:	Updated plotting part
#	User	Rev	Content
1	buchmann	1.1	#!/bin/bash
2
3			function usage {
4			echo "You can use this script to send crab jobs for scans. You can either run it directly and answer the two questions, or you can answer them directly using -n NAME -j NJOBS"
5			exit 1
6			}
7
8			function clean_file {
9			echo "Cleaning: $1"
10			if [ -e $1 ]; then
11			rm $1
12			fi
13			}
14
15			function clean {
16			clean_file "Crab_Jobs/runthis.sh"
17			clean_file "Crab_Jobs/runthis.sh"
18			clean_file "Crab_Jobs/crabconfig.cfg"
19			clean_file "Crab_Jobs/Modules.tar"
20			clean_file "Crab_Jobs/AddFiles.tar"
21			clean_file "Crab_Jobs/dummy.py"
22			}
23
24			function produce_all_plots {
25			directory=$1
26	buchmann	1.2
27			## delete previous executable so everything's up-to-date
28			clean_file "PlotScanResults.C"
29
30			cp ../Limits/PlotScanResults.C .
31			g++ PlotScanResults.C -o PlotScanResults.exec `root-config --glibs --cflags` -lTreePlayer
32			clean_file "PlotScanResults.C"
33			echo "Compiled script to produce Going to produce plots now"
34			PlotScanResults.exec $directory
35			clean_file "LOGerr.txt"
36			clean_file "Tex.txt"
37			clean_file "LOG.txt"
38	buchmann	1.1	echo "All plots have been generated. That's all, now have a look at the results :-)"
39			}
40
41			function mail_all_plots {
42			tar -cvf Plots.tar $1
43			echo "About to send all plots"
44			####### DO NOT UPLOAD THE SCRIPT IN THIS LOCATION OR MARCO WILL GET VERY,VERY,VERY MAD AT YOU
45			/shome/buchmann/material/send_mail_with_attachment.sh $1 Plots.tar
46			rm Plots.tar
47			}
48
49			name=""
50			njobs=1234567890
51			while getopts "j:n:h" OPTION
52			do
53			case $OPTION in
54			j)
55			njobs=$OPTARG
56			echo "Have received option for number of jobs. njobs=$njobs set".
57			;;
58			n)
59			name=$OPTARG
60			echo "Have received option for name. name=$name set".
61			;;
62			h)
63			usage
64			exit
65			;;
66			esac
67			done
68
69			while [ $njobs -lt 1 ] \|\| [ $njobs -gt 5000 ];
70			do
71			njobs=0
72			echo "Question 1: How many jobs would you like to split this into? (0<njobs<5000), (-j) njobs="
73			read njobs
74			done
75			echo "Accepted number of jobs: $njobs"
76
77			if [[ "$name" == "" ]]; then
78			echo ""
79			else
80			if [ -d "Crab_Jobs/$name" ]; then
81			echo "The directory identifier you provided ($name) is taken. Please try again."
82			name=""
83			else
84			echo "The directory identifier you provided ($name) is ok."
85			fi
86			fi
87
88			if [[ "$name" == "" ]]; then
89			echo "Question 2: Name for this job? [no spaces!] (-n) name="
90			read name
91			fi
92
93			if [ -d "Crab_Jobs/$name" ]; then
94			echo "The directory identifier you provided ($name) is also taken. Please start again."
95			exit -1
96			fi
97
98
99			echo "Accepted job name : $name"
100
101			echo "We have now established the number of jobs (${njobs}) as well as the jobs name (${name}) which will serve as working directory"
102			mkdir -p Crab_Jobs
103
104			eval "cat templates/runthis.sh \| sed 's/TOTNJOBS/${njobs}/' > Crab_Jobs/runthis.sh"
105			eval "cat templates/crabconfig.cfg \| sed 's/NJOBS/${njobs}/' \| sed 's/WORKINGDIR/${name}/' > Crab_Jobs/crabconfig.cfg"
106
107			echo "Have generated the run script (runthis.sh) as well as the crab configuration file (crabconfig.cfg); will now attempt to create the crab job"
108
109			cp templates/dummy.py Crab_Jobs/
110			cp templates/Fake_Makefile Makefile
111
112			tar -cvf Crab_Jobs/Modules.tar ../../Plotting/Modules/ >/dev/null 2>/dev/null
113			tar -cvf Crab_Jobs/AddFiles.tar ../Limits/TimedLimitCapsule.C >/dev/null 2>/dev/null
114			tar -rvf Crab_Jobs/AddFiles.tar ../LimitsFromSystematics/WorkerScript.C >/dev/null 2>/dev/null
115			tar -rvf Crab_Jobs/AddFiles.tar ../last_configuration.C >/dev/null 2>/dev/null
116			tar -rvf Crab_Jobs/AddFiles.tar ../Systematics/SystematicsWorkerScript.C >/dev/null 2>/dev/null
117			tar -rvf Crab_Jobs/AddFiles.tar Makefile >/dev/null 2>/dev/null
118
119			bash utils/get_latest_root_version.sh
120			rootfilename=`ls \| grep root \| grep gcc`
121			echo "Attaching root file $rootfilename to tarball ..."
122			tar -rvf Crab_Jobs/AddFiles.tar $rootfilename >/dev/null 2>/dev/null
123
124			cd Crab_Jobs && crab -create -cfg crabconfig.cfg
125			cd ../
126			rm Makefile
127
128			if [[ -e Crab_Jobs/$name ]]; then
129			echo "Crab seems to have run correctly. Will now clean up a bit."
130			rm Crab_Jobs/AddFiles.tar
131			rm Crab_Jobs/Modules.tar
132			rm Crab_Jobs/dummy.py
133			if [[ -f Crab_Jobs/dummy.pyc ]]; then
134			rm Crab_Jobs/dummy.pyc
135			fi
136			rm Crab_Jobs/crabconfig.cfg
137			rm Crab_Jobs/runthis.sh
138			else
139			echo "There seems to have been a problem while trying to generate the crab job (can't find any Crab_Jobs/$name directory); please verify that the crab environment is correctly set up. If you would like to create the crab job manually (and monitor it yourself), you can still do: \n \n cd Crab_Jobs && crab -c crabconfig.cfg -create -submit"
140			exit -1
141			fi
142
143			echo "Would you like to submit the jobs manually (e.g. for testing) [me] or would you like this script to submit the jobs and monitor them [anything else]?"
144			read answer
145			if [[ "$answer" == "me" ]]; then
146			echo "Command you might need:"
147			echo "cd Crab_Jobs && crab -c ${name} -submit"
148			echo "You're on your own now. Goodbye!"
149			exit 0
150			fi
151
152			echo "You chose to have this handled automatically. Please sit back, submitting jobs now!"
153			cd Crab_Jobs && crab -c ${name} -submit && cd ..
154			echo "Ok, everything has been submitted. Now let's check on the status ... (this will take a long time)"
155			### THIS SCRIPT REMAINS ON THE T3!!! DO NOT UPLOAD IT!
156			/shome/buchmann/material/check_status_plus_LIST.sh -r `pwd`/Crab_Jobs/${name}
157
158			echo "Now going to fuse everything related to this job: ${name}"
159			python utils/fusejobs.py -d ${name}
160			rm -r Plots/
161			source /swshare/ROOT/thisroot.sh
162	buchmann	1.2
163			rm -r Plots_${name}/
164			rm -r Plots/
165	buchmann	1.1	produce_all_plots "/scratch/`whoami`/bussolaresults/${name}.root"
166			mv Plots/ Plots_${name}/
167			mail_all_plots Plots_${name}
168			echo "Done. Your results are in Plots_${name}/ :-)"