ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/UserCode/MitProd/Processing/bin/download.sh

Comparing UserCode/MitProd/Processing/bin/download.sh (file contents):
Revision 1.2 by ceballos, Fri Dec 10 09:41:33 2010 UTC vs.
Revision 1.6 by paus, Wed Sep 28 21:02:50 2011 UTC

#	Line 7 | Line 7 | SAMPLE_LIST=$1
7		MIT_LOCATION="/pnfs/cmsaf.mit.edu/t2bat/cms/store/user/paus"
8		CERN_LOCATION="/castor/cern.ch/user/p/paus"
9
10	+	# start next request
11	+	remainingMax=0
12	+
13		# health checks
14		if [ ".$1" == "." ]
15		then
#	Line 22 | Line 25 | echo " ==== Download queue from $SAMPLE_
25		echo ""
26
27		# check that queues are really empty
28	+	dataset="XXX"
29		startTime=$(date +%s)
30		nowTime=$(date +%s); duration=$(($nowTime - $startTime))
31	<	jobs=`condor_q -global cmsprod | grep downloadFiles.sh | wc -l`
32	<	while [ "$jobs" != "0" ]
31	>	jobs=`condor_q -global $USER -format "%d " ClusterId -format "%s " Cmd -format "%s \n" Args | grep $dataset | grep downloadFiles.sh | wc -l`
32	>	while [ $jobs -gt $remainingMax ]
33		do
34		echo " waiting since  $duration sec  == condor queue has  $jobs jobs  left"; sleep 60; echo ""
35	<	jobs=`condor_q -global cmsprod | grep downloadFiles.sh | wc -l`
35	>	jobs=`condor_q -global $USER $format | grep $dataset | grep downloadFiles.sh | wc -l`
36	>	#jobs=`condor_q -global $USER | grep downloadFiles.sh | wc -l`
37		nowTime=$(date +%s); duration=$(($nowTime - $startTime))
38		done
39	<	echo " Queues are empty ($jobs) Let's get started."
39	>	echo " Queues are close to empty ($jobs) -- Let's get started."
40		echo ""
41
42		# all right, ready to download
#	Line 39 | Line 44 | nSamples=`cat $SAMPLE_LIST | wc -l | cut
44		i=1
45		while [ $i -le $nSamples ]
46		do
47	+	startTime=$(date +%s)
48
49		line=`head -$i $SAMPLE_LIST | tail -1`
50		if [ "`echo $line | grep ^#`" != "" ]
#	Line 48 | Line 54 | do
54		fi
55
56		# define some quantities
57	+	baseDir=`echo $line | tr -s ' ' | cut -d ' ' -f 1`
58		book=`echo $line | tr -s ' ' | cut -d ' ' -f 2`
59		version=`basename $book`
60		dataset=`echo $line | tr -s ' ' | cut -d ' ' -f 3`
61	+	targetDir=`echo $line | tr -s ' ' | cut -d ' ' -f 4`
62	+
63	+	# stagein the sample if it is at CERN
64	+	if [ "`echo $baseDir | grep /castor/cern.ch`" != "" ]
65	+	then
66	+	echo "  ssh paus@lxplus.cern.ch ./stageSample.py --dataDir=$baseDir/$book/$dataset"
67	+	ssh paus@lxplus.cern.ch ./stageSample.py --dataDir=$baseDir/$book/$dataset
68	+	fi
69
70	<	#echo " downloadSample.sh $line"
71	<	#mkdir -p /mnt/hadoop/cmsprod/$book/$dataset
57	<	mkdir -p /data/blue/cmsprod/$book/$dataset
70	>	# download the sample
71	>	echo " downloadSample.sh $line"
72		downloadSample.sh $line
73
74	+	# go into waiting loop
75		nowTime=$(date +%s); duration=$(($nowTime - $startTime))
76	<	jobs=`condor_q -global cmsprod | grep downloadFiles.sh | wc -l`
77	<	while [ "$jobs" != "0" ]
76	>	jobs=`condor_q -global $USER -format "%d " ClusterId -format "%s " Cmd -format "%s \n" Args | grep $dataset | grep downloadFiles.sh | wc -l`
77	>	while [ $jobs -gt $remainingMax ]
78		do
79		echo " waiting since  $duration sec  == condor queue has  $jobs jobs  left"; sleep 60; echo ""
80	<	jobs=`condor_q -global cmsprod | grep downloadFiles.sh | wc -l`
80	>	jobs=`condor_q -global $USER -format "%d " ClusterId -format "%s " Cmd -format "%s \n" Args | grep $dataset | grep downloadFiles.sh | wc -l`
81		nowTime=$(date +%s); duration=$(($nowTime - $startTime))
82		done
83		echo " Queues are empty ($jobs) --> cleaning up and making catalogs."
84		echo ""
85
86		# remove zero length files
87	<	echo "removeZeroLengthFiles.sh /mnt/hadoop/cmsprod/$book/$dataset"
88	<	removeZeroLengthFiles.sh /mnt/hadoop/cmsprod/$book/$dataset
87	>	echo "removeZeroLengthFiles.sh $targetDir/$book/$dataset"
88	>	removeZeroLengthFiles.sh $targetDir/$book/$dataset
89
90		# finally make the corresponding catalog
91	<	#echo "catalog.sh -ceg $version $dataset --retry /mnt/hadoop/cmsprod"
92	<	catalog.sh -ceg $version $dataset --retry /mnt/hadoop/cmsprod
91	>	echo "catalog.sh -ceg $version $dataset --retry $targetDir"
92	>	catalog.sh -ceg $version $dataset --retry $targetDir
93
94		i=$(( $i+1 ))
95

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines