| 1 |
|
#!/bin/bash |
| 2 |
|
# This script submitts all jobs to naf, which are defined in dataset |
| 3 |
|
|
| 4 |
< |
datasets=( srm://dcache-se-cms.desy.de:8443/srm/managerv2?SFN=/pnfs/desy.de/cms/tier2/store/user/jschulz/nTuples/QCD_250-500_V01/ |
| 5 |
< |
srm://dcache-se-cms.desy.de:8443/srm/managerv2?SFN=/pnfs/desy.de/cms/tier2/store/user/jschulz/nTuples/QCD_500-1000_V01/ |
| 6 |
< |
srm://dcache-se-cms.desy.de:8443/srm/managerv2?SFN=/pnfs/desy.de/cms/tier2/store/user/jschulz/nTuples/QCD_1000-inf_V01/ |
| 7 |
< |
srm://dcache-se-cms.desy.de:8443/srm/managerv2?SFN=/pnfs/desy.de/cms/tier2/store/user/jschulz/nTuples/WJets_V01/ |
| 8 |
< |
srm://dcache-se-cms.desy.de:8443/srm/managerv2?SFN=/pnfs/desy.de/cms/tier2/store/user/jschulz/nTuples/GVJets_V01/ |
| 9 |
< |
srm://dcache-se-cms.desy.de:8443/srm/managerv2?SFN=/pnfs/desy.de/cms/tier2/store/user/jschulz/nTuples/TTJets_V01/ ) |
| 10 |
< |
#srm://dcache-se-cms.desy.de:8443/srm/managerv2?SFN=/pnfs/desy.de/cms/tier2/store/user/jschulz/nTuples/GJets_V01/ ) |
| 4 |
> |
datasets=( |
| 5 |
> |
# srm://dcache-se-cms.desy.de:8443/srm/managerv2?SFN=/pnfs/desy.de/cms/tier2/store/user/jschulz/nTuples/QCD_250-500_V01/ |
| 6 |
> |
# srm://dcache-se-cms.desy.de:8443/srm/managerv2?SFN=/pnfs/desy.de/cms/tier2/store/user/jschulz/nTuples/QCD_500-1000_V01/ |
| 7 |
> |
# srm://dcache-se-cms.desy.de:8443/srm/managerv2?SFN=/pnfs/desy.de/cms/tier2/store/user/jschulz/nTuples/QCD_1000-inf_V01/ |
| 8 |
> |
# srm://dcache-se-cms.desy.de:8443/srm/managerv2?SFN=/pnfs/desy.de/cms/tier2/store/user/jschulz/nTuples/GJets_V01/ |
| 9 |
> |
# srm://dcache-se-cms.desy.de:8443/srm/managerv2?SFN=/pnfs/desy.de/cms/tier2/store/user/jschulz/nTuples/WJets_V01/ |
| 10 |
> |
# srm://dcache-se-cms.desy.de:8443/srm/managerv2?SFN=/pnfs/desy.de/cms/tier2/store/user/jschulz/nTuples/TTJets_V01/ |
| 11 |
> |
srm://dcache-se-cms.desy.de:8443/srm/managerv2?SFN=/pnfs/desy.de/cms/tier2/store/user/jschulz/nTuples/DY_to_ee/ |
| 12 |
> |
# srm://dcache-se-cms.desy.de:8443/srm/managerv2?SFN=/pnfs/desy.de/cms/tier2/store/user/jschulz/nTuples/GVJets_V01/ |
| 13 |
> |
) |
| 14 |
|
# settings |
| 15 |
< |
version="00" |
| 15 |
> |
version="14" |
| 16 |
> |
outputFolder=/scratch/hh/dust/naf/cms/user/kiesel |
| 17 |
|
files_per_job=20 |
| 18 |
|
|
| 19 |
+ |
fetchScript=getOutput_${version}.sh |
| 20 |
+ |
echo "#!/bin/bash -e" > $fetchScript |
| 21 |
+ |
chmod +x $fetchScript |
| 22 |
+ |
|
| 23 |
|
for dataset in "${datasets[@]}"; do |
| 24 |
|
|
| 25 |
|
# get folder name as best description for job |
| 27 |
|
# Since root can't handle -, it will be substituted to _ |
| 28 |
|
job_name=$(echo $job_name|sed 's/-/_/g') |
| 29 |
|
|
| 30 |
< |
files=( $(srmls $dataset|grep root|awk '{print $2 }') ) |
| 30 |
> |
files=( $(srmls -offset 0 -count 999 $dataset|grep root|awk '{print $2 }') ) |
| 31 |
> |
# Ugly hack to get more than 1000 files (up to 2000). |
| 32 |
> |
files+=( $(srmls -offset 1000 -count 1999 $dataset|grep root|awk '{print $2 }') ) |
| 33 |
|
number_of_jobs=$(expr ${#files[@]} / $files_per_job + 1 ) |
| 34 |
|
|
| 35 |
|
# just for testing purpurse |
| 46 |
|
files_to_submit=$files_to_submit"dcap://dcache-cms-dcap.desy.de"${files[$i]}" " |
| 47 |
|
fi |
| 48 |
|
done # files for one job |
| 49 |
< |
outputFileName=/scratch/hh/dust/naf/cms/user/kiesel/${jobPrefix}_tree.root |
| 49 |
> |
outputFileName=$outputFolder/${jobPrefix}_tree.root |
| 50 |
|
|
| 51 |
|
# now that we have all information, get data |
| 52 |
|
script=$jobPrefix.sh |
| 54 |
|
echo export SCRAM_ARCH="slc5_amd64_gcc462" > $script |
| 55 |
|
echo export VO_CMS_SW_DIR=/cvmfs/cms.cern.ch >> $script |
| 56 |
|
echo source $VO_CMS_SW_DIR/cmsset_default.sh >> $script |
| 47 |
– |
echo ini glite >> $script |
| 57 |
|
echo cd $HOME/CMSSW_5_3_8/src >> $script |
| 49 |
– |
echo ini cmssw >> $script |
| 58 |
|
echo cmsenv >> $script |
| 59 |
|
echo cd $HOME/treeWriter >> $script |
| 60 |
+ |
echo make >> $script |
| 61 |
|
|
| 62 |
|
echo ./executable $outputFileName $files_to_submit >> $script |
| 63 |
|
chmod +x $script |
| 64 |
< |
qsub -b y -j y -l h_cpu=07:00:00 -l site=hh `pwd`/$jobPrefix.sh |
| 65 |
< |
echo |
| 64 |
> |
qsub -b y -j y -l h_cpu=7:00:00 -l site=hh `pwd`/$jobPrefix.sh |
| 65 |
> |
# i: merge stdout and stderr |
| 66 |
> |
# o: log file |
| 67 |
|
done # all jobs |
| 68 |
+ |
echo "rm ${job_name}.${version}__*.sh" >> $fetchScript |
| 69 |
+ |
echo "rm ~/${job_name}.${version}__*.sh.o*" >> $fetchScript |
| 70 |
+ |
echo "hadd $outputFolder/${job_name}.${version}_tree.root $outputFolder/${job_name}.${version}__*.root" >> $fetchScript |
| 71 |
+ |
echo "rm $outputFolder/${job_name}.${version}__*.root" >> $fetchScript |
| 72 |
|
done #dataset |
