DistributedModelCalculations/CBAFviaCRAB/SendCrabJob.sh

#!/bin/bash

function usage {
 echo "You can use this script to send crab jobs for scans. You can either run it directly and answer the two questions, or you can answer them directly using -n NAME -j NJOBS"
 exit 1
}
 
function clean_file {
 echo "Cleaning: $1" 
 if [ -e $1 ]; then
     rm $1
 fi
}

function clean {
 clean_file "Crab_Jobs/runthis.sh"
 clean_file "Crab_Jobs/runthis.sh"
 clean_file "Crab_Jobs/crabconfig.cfg"
 clean_file "Crab_Jobs/Modules.tar"
 clean_file "Crab_Jobs/AddFiles.tar"
 clean_file "Crab_Jobs/dummy.py"
}

function produce_all_plots {
 directory=$1

 ## delete previous executable so everything's up-to-date
 clean_file "PlotScanResults.C"

 cp ../Limits/PlotScanResults.C .
 g++ PlotScanResults.C -o PlotScanResults.exec `root-config --glibs --cflags` -lTreePlayer
 clean_file "PlotScanResults.C"
 echo "Compiled script to produce Going to produce plots now"
 PlotScanResults.exec $directory
 clean_file "LOGerr.txt"
 clean_file "Tex.txt"
 clean_file "LOG.txt"
 echo "All plots have been generated. That's all, now have a look at the results :-)"
}

function mail_all_plots {
        tar -cvf Plots.tar $1
        echo "About to send all plots"
####### DO NOT UPLOAD THE SCRIPT IN THIS LOCATION OR MARCO WILL GET *VERY,VERY,VERY MAD* AT YOU
        /shome/buchmann/material/send_mail_with_attachment.sh $1 Plots.tar
        rm Plots.tar
}

 name=""
 njobs=1234567890
 while getopts "j:n:h" OPTION
 do
   case $OPTION in
     j)
       njobs=$OPTARG
       echo "Have received option for number of jobs. njobs=$njobs set".
       ;;
     n)
       name=$OPTARG
       echo "Have received option for name. name=$name set".
       ;;
     h)
       usage
       exit
       ;;
    esac
 done
        
 while [ $njobs -lt 1 ] || [ $njobs -gt 5000 ];
 do
    njobs=0
    echo "Question 1: How many jobs would you like to split this into? (0<njobs<5000), (-j) njobs="
    read njobs
 done
 echo "Accepted number of jobs: $njobs"
 
 if [[ "$name" == "" ]]; then 
    echo ""
    else
    if [ -d "Crab_Jobs/$name" ]; then
     echo "The directory identifier you provided ($name) is taken. Please try again."
     name=""
    else 
     echo "The directory identifier you provided ($name) is ok."
    fi
fi

 if [[ "$name" == "" ]]; then
    echo "Question 2: Name for this job? [no spaces!] (-n) name="
    read name
 fi

 if [ -d "Crab_Jobs/$name" ]; then
     echo "The directory identifier you provided ($name) is also taken. Please start again."
     exit -1
 fi 


 echo "Accepted job name : $name"

 echo "We have now established the number of jobs (${njobs}) as well as the jobs name (${name}) which will serve as working directory"
 mkdir -p Crab_Jobs

 eval "cat templates/runthis.sh | sed 's/TOTNJOBS/${njobs}/' > Crab_Jobs/runthis.sh"
 eval "cat templates/crabconfig.cfg | sed 's/NJOBS/${njobs}/' | sed 's/WORKINGDIR/${name}/' > Crab_Jobs/crabconfig.cfg"

 echo "Have generated the run script (runthis.sh) as well as the crab configuration file (crabconfig.cfg); will now attempt to create the crab job"
 
 cp templates/dummy.py Crab_Jobs/
 cp templates/Fake_Makefile Makefile 

 tar -cvf Crab_Jobs/Modules.tar ../../Plotting/Modules/ >/dev/null 2>/dev/null
 tar -cvf Crab_Jobs/AddFiles.tar ../Limits/TimedLimitCapsule.C >/dev/null 2>/dev/null
 tar -rvf Crab_Jobs/AddFiles.tar ../LimitsFromSystematics/WorkerScript.C >/dev/null 2>/dev/null
 tar -rvf Crab_Jobs/AddFiles.tar ../last_configuration.C >/dev/null 2>/dev/null
 tar -rvf Crab_Jobs/AddFiles.tar ../Systematics/SystematicsWorkerScript.C >/dev/null 2>/dev/null 
 tar -rvf Crab_Jobs/AddFiles.tar Makefile >/dev/null 2>/dev/null

 bash utils/get_latest_root_version.sh
 rootfilename=`ls | grep root | grep gcc`
 echo "Attaching root file $rootfilename to tarball ..." 
 tar -rvf Crab_Jobs/AddFiles.tar $rootfilename >/dev/null 2>/dev/null

 cd Crab_Jobs && crab -create -cfg crabconfig.cfg
 cd ../
 rm Makefile

 if [[ -e Crab_Jobs/$name ]]; then 
    echo "Crab seems to have run correctly. Will now clean up a bit."
    rm Crab_Jobs/AddFiles.tar
    rm Crab_Jobs/Modules.tar
    rm Crab_Jobs/dummy.py
    if [[ -f Crab_Jobs/dummy.pyc ]]; then 
        rm Crab_Jobs/dummy.pyc
    fi
    rm Crab_Jobs/crabconfig.cfg
    rm Crab_Jobs/runthis.sh
 else
   echo "There seems to have been a problem while trying to generate the crab job (can't find any Crab_Jobs/$name directory); please verify that the crab environment is correctly set up. If you would like to create the crab job manually (and monitor it yourself), you can still do: \n \n cd Crab_Jobs && crab -c crabconfig.cfg -create -submit"
    exit -1
 fi

 echo "Would you like to submit the jobs manually (e.g. for testing) [me] or would you like this script to submit the jobs and monitor them [anything else]?"
 read answer
 if [[ "$answer" == "me" ]]; then
         echo "Command you might need:"
         echo "cd Crab_Jobs && crab -c ${name} -submit"
         echo "You're on your own now. Goodbye!"
         exit 0
 fi

 echo "You chose to have this handled automatically. Please sit back, submitting jobs now!"
 cd Crab_Jobs && crab -c ${name} -submit && cd ..
 echo "Ok, everything has been submitted. Now let's check on the status ... (this will take a long time)"
### THIS SCRIPT REMAINS ON THE T3!!! DO NOT UPLOAD IT!
/shome/buchmann/material/check_status_plus_LIST.sh -r `pwd`/Crab_Jobs/${name}

 echo "Now going to fuse everything related to this job: ${name}"
 python utils/fusejobs.py -d ${name}
 rm -r Plots/
 source /swshare/ROOT/thisroot.sh

 rm -r Plots_${name}/
 rm -r Plots/
 produce_all_plots "/scratch/`whoami`/bussolaresults/${name}.root"
 mv Plots/ Plots_${name}/
 mail_all_plots Plots_${name}
 echo "Done. Your results are in Plots_${name}/ :-)"
Revision:	1.2
Committed:	Thu Jan 19 14:14:36 2012 UTC (13 years, 3 months ago) by buchmann
Content type:	application/x-sh
Branch:	MAIN
CVS Tags:	cbaf_4_98ifb_paper, cbaf_4p7ifb, HEAD
Changes since 1.1:	+15 -7 lines
Log Message:	Updated plotting part
#	Content
1	#!/bin/bash
2
3	function usage {
4	echo "You can use this script to send crab jobs for scans. You can either run it directly and answer the two questions, or you can answer them directly using -n NAME -j NJOBS"
5	exit 1
6	}
7
8	function clean_file {
9	echo "Cleaning: $1"
10	if [ -e $1 ]; then
11	rm $1
12	fi
13	}
14
15	function clean {
16	clean_file "Crab_Jobs/runthis.sh"
17	clean_file "Crab_Jobs/runthis.sh"
18	clean_file "Crab_Jobs/crabconfig.cfg"
19	clean_file "Crab_Jobs/Modules.tar"
20	clean_file "Crab_Jobs/AddFiles.tar"
21	clean_file "Crab_Jobs/dummy.py"
22	}
23
24	function produce_all_plots {
25	directory=$1
26
27	## delete previous executable so everything's up-to-date
28	clean_file "PlotScanResults.C"
29
30	cp ../Limits/PlotScanResults.C .
31	g++ PlotScanResults.C -o PlotScanResults.exec `root-config --glibs --cflags` -lTreePlayer
32	clean_file "PlotScanResults.C"
33	echo "Compiled script to produce Going to produce plots now"
34	PlotScanResults.exec $directory
35	clean_file "LOGerr.txt"
36	clean_file "Tex.txt"
37	clean_file "LOG.txt"
38	echo "All plots have been generated. That's all, now have a look at the results :-)"
39	}
40
41	function mail_all_plots {
42	tar -cvf Plots.tar $1
43	echo "About to send all plots"
44	####### DO NOT UPLOAD THE SCRIPT IN THIS LOCATION OR MARCO WILL GET VERY,VERY,VERY MAD AT YOU
45	/shome/buchmann/material/send_mail_with_attachment.sh $1 Plots.tar
46	rm Plots.tar
47	}
48
49	name=""
50	njobs=1234567890
51	while getopts "j:n:h" OPTION
52	do
53	case $OPTION in
54	j)
55	njobs=$OPTARG
56	echo "Have received option for number of jobs. njobs=$njobs set".
57	;;
58	n)
59	name=$OPTARG
60	echo "Have received option for name. name=$name set".
61	;;
62	h)
63	usage
64	exit
65	;;
66	esac
67	done
68
69	while [ $njobs -lt 1 ] \|\| [ $njobs -gt 5000 ];
70	do
71	njobs=0
72	echo "Question 1: How many jobs would you like to split this into? (0<njobs<5000), (-j) njobs="
73	read njobs
74	done
75	echo "Accepted number of jobs: $njobs"
76
77	if [[ "$name" == "" ]]; then
78	echo ""
79	else
80	if [ -d "Crab_Jobs/$name" ]; then
81	echo "The directory identifier you provided ($name) is taken. Please try again."
82	name=""
83	else
84	echo "The directory identifier you provided ($name) is ok."
85	fi
86	fi
87
88	if [[ "$name" == "" ]]; then
89	echo "Question 2: Name for this job? [no spaces!] (-n) name="
90	read name
91	fi
92
93	if [ -d "Crab_Jobs/$name" ]; then
94	echo "The directory identifier you provided ($name) is also taken. Please start again."
95	exit -1
96	fi
97
98
99	echo "Accepted job name : $name"
100
101	echo "We have now established the number of jobs (${njobs}) as well as the jobs name (${name}) which will serve as working directory"
102	mkdir -p Crab_Jobs
103
104	eval "cat templates/runthis.sh \| sed 's/TOTNJOBS/${njobs}/' > Crab_Jobs/runthis.sh"
105	eval "cat templates/crabconfig.cfg \| sed 's/NJOBS/${njobs}/' \| sed 's/WORKINGDIR/${name}/' > Crab_Jobs/crabconfig.cfg"
106
107	echo "Have generated the run script (runthis.sh) as well as the crab configuration file (crabconfig.cfg); will now attempt to create the crab job"
108
109	cp templates/dummy.py Crab_Jobs/
110	cp templates/Fake_Makefile Makefile
111
112	tar -cvf Crab_Jobs/Modules.tar ../../Plotting/Modules/ >/dev/null 2>/dev/null
113	tar -cvf Crab_Jobs/AddFiles.tar ../Limits/TimedLimitCapsule.C >/dev/null 2>/dev/null
114	tar -rvf Crab_Jobs/AddFiles.tar ../LimitsFromSystematics/WorkerScript.C >/dev/null 2>/dev/null
115	tar -rvf Crab_Jobs/AddFiles.tar ../last_configuration.C >/dev/null 2>/dev/null
116	tar -rvf Crab_Jobs/AddFiles.tar ../Systematics/SystematicsWorkerScript.C >/dev/null 2>/dev/null
117	tar -rvf Crab_Jobs/AddFiles.tar Makefile >/dev/null 2>/dev/null
118
119	bash utils/get_latest_root_version.sh
120	rootfilename=`ls \| grep root \| grep gcc`
121	echo "Attaching root file $rootfilename to tarball ..."
122	tar -rvf Crab_Jobs/AddFiles.tar $rootfilename >/dev/null 2>/dev/null
123
124	cd Crab_Jobs && crab -create -cfg crabconfig.cfg
125	cd ../
126	rm Makefile
127
128	if [[ -e Crab_Jobs/$name ]]; then
129	echo "Crab seems to have run correctly. Will now clean up a bit."
130	rm Crab_Jobs/AddFiles.tar
131	rm Crab_Jobs/Modules.tar
132	rm Crab_Jobs/dummy.py
133	if [[ -f Crab_Jobs/dummy.pyc ]]; then
134	rm Crab_Jobs/dummy.pyc
135	fi
136	rm Crab_Jobs/crabconfig.cfg
137	rm Crab_Jobs/runthis.sh
138	else
139	echo "There seems to have been a problem while trying to generate the crab job (can't find any Crab_Jobs/$name directory); please verify that the crab environment is correctly set up. If you would like to create the crab job manually (and monitor it yourself), you can still do: \n \n cd Crab_Jobs && crab -c crabconfig.cfg -create -submit"
140	exit -1
141	fi
142
143	echo "Would you like to submit the jobs manually (e.g. for testing) [me] or would you like this script to submit the jobs and monitor them [anything else]?"
144	read answer
145	if [[ "$answer" == "me" ]]; then
146	echo "Command you might need:"
147	echo "cd Crab_Jobs && crab -c ${name} -submit"
148	echo "You're on your own now. Goodbye!"
149	exit 0
150	fi
151
152	echo "You chose to have this handled automatically. Please sit back, submitting jobs now!"
153	cd Crab_Jobs && crab -c ${name} -submit && cd ..
154	echo "Ok, everything has been submitted. Now let's check on the status ... (this will take a long time)"
155	### THIS SCRIPT REMAINS ON THE T3!!! DO NOT UPLOAD IT!
156	/shome/buchmann/material/check_status_plus_LIST.sh -r `pwd`/Crab_Jobs/${name}
157
158	echo "Now going to fuse everything related to this job: ${name}"
159	python utils/fusejobs.py -d ${name}
160	rm -r Plots/
161	source /swshare/ROOT/thisroot.sh
162
163	rm -r Plots_${name}/
164	rm -r Plots/
165	produce_all_plots "/scratch/`whoami`/bussolaresults/${name}.root"
166	mv Plots/ Plots_${name}/
167	mail_all_plots Plots_${name}
168	echo "Done. Your results are in Plots_${name}/ :-)"