ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/UserCode/MitProd/Processing/bin/download.sh

Comparing UserCode/MitProd/Processing/bin/download.sh (file contents):
Revision 1.3 by paus, Thu Jan 6 22:30:07 2011 UTC vs.
Revision 1.5 by paus, Mon Sep 19 21:45:40 2011 UTC

#	Line 7 | Line 7 | SAMPLE_LIST=$1
7		MIT_LOCATION="/pnfs/cmsaf.mit.edu/t2bat/cms/store/user/paus"
8		CERN_LOCATION="/castor/cern.ch/user/p/paus"
9
10	+	# start next request
11	+	remainingMax=0
12	+
13		# health checks
14		if [ ".$1" == "." ]
15		then
#	Line 22 | Line 25 | echo " ==== Download queue from $SAMPLE_
25		echo ""
26
27		# check that queues are really empty
28	+	dataset="XXX"
29		startTime=$(date +%s)
30		nowTime=$(date +%s); duration=$(($nowTime - $startTime))
31	<	jobs=`condor_q -global $USER | grep downloadFiles.sh | wc -l`
32	<	while [ $jobs -gt 15 ]
31	>	format='-format "%d " ClusterId -format "%s " Cmd -format "%s \n" Args'
32	>	jobs=`condor_q -global $USER $format | grep $dataset | grep downloadFiles.sh | wc -l`
33	>	#jobs=`condor_q -global $USER | grep downloadFiles.sh | wc -l`
34	>	while [ $jobs -gt $remainingMax ]
35		do
36		echo " waiting since  $duration sec  == condor queue has  $jobs jobs  left"; sleep 60; echo ""
37	<	jobs=`condor_q -global $USER | grep downloadFiles.sh | wc -l`
37	>	jobs=`condor_q -global $USER $format | grep $dataset | grep downloadFiles.sh | wc -l`
38	>	#jobs=`condor_q -global $USER | grep downloadFiles.sh | wc -l`
39		nowTime=$(date +%s); duration=$(($nowTime - $startTime))
40		done
41		echo " Queues are close to empty ($jobs) -- Let's get started."
#	Line 39 | Line 46 | nSamples=`cat $SAMPLE_LIST | wc -l | cut
46		i=1
47		while [ $i -le $nSamples ]
48		do
49	+	startTime=$(date +%s)
50
51		line=`head -$i $SAMPLE_LIST | tail -1`
52		if [ "`echo $line | grep ^#`" != "" ]
#	Line 52 | Line 60 | do
60		book=`echo $line | tr -s ' ' | cut -d ' ' -f 2`
61		version=`basename $book`
62		dataset=`echo $line | tr -s ' ' | cut -d ' ' -f 3`
63	+	targetDir=`echo $line | tr -s ' ' | cut -d ' ' -f 4`
64
65		# stagein the sample if it is at CERN
66		if [ "`echo $baseDir | grep /castor/cern.ch`" != "" ]
67		then
68	+	echo "  ssh paus@lxplus.cern.ch ./stageSample.py --dataDir=$baseDir/$book/$dataset"
69		ssh paus@lxplus.cern.ch ./stageSample.py --dataDir=$baseDir/$book/$dataset
70		fi
71
#	Line 65 | Line 75 | do
75
76		# go into waiting loop
77		nowTime=$(date +%s); duration=$(($nowTime - $startTime))
78	<	jobs=`condor_q -global $USER | grep downloadFiles.sh | wc -l`
79	<	while [ $jobs -gt 15 ]
78	>	#jobs=`condor_q -global $USER | grep downloadFiles.sh | wc -l`
79	>	jobs=`condor_q -global $USER $format | grep $dataset | grep downloadFiles.sh | wc -l`
80	>	while [ $jobs -gt $remainingMax ]
81		do
82		echo " waiting since  $duration sec  == condor queue has  $jobs jobs  left"; sleep 60; echo ""
83	<	jobs=`condor_q -global $USER | grep downloadFiles.sh | wc -l`
83	>	#jobs=`condor_q -global $USER | grep downloadFiles.sh | wc -l`
84	>	jobs=`condor_q -global $USER $format | grep $dataset | grep downloadFiles.sh | wc -l`
85		nowTime=$(date +%s); duration=$(($nowTime - $startTime))
86		done
87		echo " Queues are empty ($jobs) --> cleaning up and making catalogs."
88		echo ""
89
90		# remove zero length files
91	<	echo "removeZeroLengthFiles.sh /mnt/hadoop/cmsprod/$book/$dataset"
92	<	removeZeroLengthFiles.sh /mnt/hadoop/cmsprod/$book/$dataset
91	>	echo "removeZeroLengthFiles.sh $targetDir/$book/$dataset"
92	>	removeZeroLengthFiles.sh $targetDir/$book/$dataset
93
94		# finally make the corresponding catalog
95	<	echo "catalog.sh -ceg $version $dataset --retry /mnt/hadoop/cmsprod"
96	<	catalog.sh -ceg $version $dataset --retry /mnt/hadoop/cmsprod
95	>	echo "catalog.sh -ceg $version $dataset --retry $targetDir"
96	>	catalog.sh -ceg $version $dataset --retry $targetDir
97
98		i=$(( $i+1 ))
99

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines