Processing/bin/download.sh

#!/bin/bash
#---------------------------------------------------------------------------------------------------
# Go through a number of download requests
#---------------------------------------------------------------------------------------------------
H=`basename $0`
SAMPLE_LIST=$1

# start next request
remainingMax=0

# health checks
if [ ".$1" == "." ]
then
  echo "";
  echo "usage: $H  <sampleList>";
  echo ""
  exit 1
fi

# say what we do
echo ""
echo " ==== Download queue from $SAMPLE_LIST ===="
echo ""

# check that queues are really empty
dataset="XXX"
startTime=$(date +%s)
nowTime=$(date +%s); duration=$(($nowTime - $startTime))
jobs=`condor_q -global $USER -format "%d " ClusterId -format "%s " Cmd -format "%s \n" Args | grep $dataset | grep downloadFiles.sh | wc -l`
while [ $jobs -gt $remainingMax ]
do
  echo " waiting since  $duration sec  == condor queue has  $jobs jobs  left"; sleep 60; echo ""
  jobs=`condor_q -global $USER $format | grep $dataset | grep downloadFiles.sh | wc -l`
  #jobs=`condor_q -global $USER | grep downloadFiles.sh | wc -l`
  nowTime=$(date +%s); duration=$(($nowTime - $startTime))
done
echo " Queues are close to empty ($jobs) -- Let's get started."
echo ""

# all right, ready to download
nSamples=`cat $SAMPLE_LIST | wc -l | cut -d ' ' -f1`
i=1
while [ $i -le $nSamples ]
do
  startTime=$(date +%s)

  line=`head -$i $SAMPLE_LIST | tail -1`
  if [ "`echo $line | grep ^#`" != "" ]
  then
    i=$(( $i+1 ))
    continue
  fi

  # define some quantities
  baseDir=`echo $line | tr -s ' ' | cut -d ' ' -f 1`
  book=`echo $line | tr -s ' ' | cut -d ' ' -f 2`
  version=`basename $book`
  dataset=`echo $line | tr -s ' ' | cut -d ' ' -f 3`
  targetDir=`echo $line | tr -s ' ' | cut -d ' ' -f 4`

  # stagein the sample if it is at CERN
  if [ "`echo $baseDir | grep /castor/cern.ch`" != "" ]
  then
    echo "  ssh paus@lxplus.cern.ch ./stageSample.py --dataDir=$baseDir/$book/$dataset"
    ssh paus@lxplus.cern.ch ./stageSample.py --dataDir=$baseDir/$book/$dataset
  fi

  # download the sample
  echo " downloadSample.sh $line"
  downloadSample.sh $line
  
  # go into waiting loop
  nowTime=$(date +%s); duration=$(($nowTime - $startTime))
  jobs=`condor_q -global $USER -format "%d " ClusterId -format "%s " Cmd -format "%s \n" Args | grep $dataset | grep downloadFiles.sh | wc -l`
  while [ $jobs -gt $remainingMax ]
  do
    echo " waiting since  $duration sec  == condor queue has  $jobs jobs  left"; sleep 60; echo ""
    jobs=`condor_q -global $USER -format "%d " ClusterId -format "%s " Cmd -format "%s \n" Args | grep $dataset | grep downloadFiles.sh | wc -l`
    nowTime=$(date +%s); duration=$(($nowTime - $startTime))
  done
  echo " Queues are empty ($jobs) --> cleaning up and making catalogs."
  echo ""
  
  # remove zero length files
  echo "removeZeroLengthFiles.sh $targetDir/$book/$dataset"
  removeZeroLengthFiles.sh $targetDir/$book/$dataset
  
  # finally make the corresponding catalog
  echo "catalog.sh -ceg $version $dataset --retry $targetDir"
  catalog.sh -ceg $version $dataset --retry $targetDir

  i=$(( $i+1 ))

done

exit 0
Revision:	1.7
Committed:	Wed Oct 19 10:43:11 2011 UTC (13 years, 6 months ago) by paus
Content type:	application/x-sh
Branch:	MAIN
Changes since 1.6:	+0 -2 lines
Log Message:	Updates realted to hadoop move of Tier-2.
#	User	Rev	Content
1	paus	1.1	#!/bin/bash
2			#---------------------------------------------------------------------------------------------------
3			# Go through a number of download requests
4			#---------------------------------------------------------------------------------------------------
5			H=`basename $0`
6			SAMPLE_LIST=$1
7
8	paus	1.4	# start next request
9			remainingMax=0
10
11	paus	1.1	# health checks
12			if [ ".$1" == "." ]
13			then
14			echo "";
15			echo "usage: $H <sampleList>";
16			echo ""
17			exit 1
18			fi
19
20			# say what we do
21			echo ""
22			echo " ==== Download queue from $SAMPLE_LIST ===="
23			echo ""
24
25			# check that queues are really empty
26	paus	1.4	dataset="XXX"
27	paus	1.1	startTime=$(date +%s)
28			nowTime=$(date +%s); duration=$(($nowTime - $startTime))
29	paus	1.6	jobs=`condor_q -global $USER -format "%d " ClusterId -format "%s " Cmd -format "%s \n" Args \| grep $dataset \| grep downloadFiles.sh \| wc -l`
30	paus	1.4	while [ $jobs -gt $remainingMax ]
31	paus	1.1	do
32			echo " waiting since $duration sec == condor queue has $jobs jobs left"; sleep 60; echo ""
33	paus	1.5	jobs=`condor_q -global $USER $format \| grep $dataset \| grep downloadFiles.sh \| wc -l`
34	paus	1.4	#jobs=`condor_q -global $USER \| grep downloadFiles.sh \| wc -l`
35	paus	1.1	nowTime=$(date +%s); duration=$(($nowTime - $startTime))
36			done
37	paus	1.3	echo " Queues are close to empty ($jobs) -- Let's get started."
38	paus	1.1	echo ""
39
40			# all right, ready to download
41			nSamples=`cat $SAMPLE_LIST \| wc -l \| cut -d ' ' -f1`
42			i=1
43			while [ $i -le $nSamples ]
44			do
45	paus	1.5	startTime=$(date +%s)
46	paus	1.1
47			line=`head -$i $SAMPLE_LIST \| tail -1`
48			if [ "`echo $line \| grep ^#`" != "" ]
49			then
50			i=$(( $i+1 ))
51			continue
52			fi
53
54			# define some quantities
55	paus	1.3	baseDir=`echo $line \| tr -s ' ' \| cut -d ' ' -f 1`
56	paus	1.1	book=`echo $line \| tr -s ' ' \| cut -d ' ' -f 2`
57			version=`basename $book`
58			dataset=`echo $line \| tr -s ' ' \| cut -d ' ' -f 3`
59	paus	1.4	targetDir=`echo $line \| tr -s ' ' \| cut -d ' ' -f 4`
60	paus	1.1
61	paus	1.3	# stagein the sample if it is at CERN
62			if [ "`echo $baseDir \| grep /castor/cern.ch`" != "" ]
63			then
64	paus	1.4	echo " ssh paus@lxplus.cern.ch ./stageSample.py --dataDir=$baseDir/$book/$dataset"
65	paus	1.3	ssh paus@lxplus.cern.ch ./stageSample.py --dataDir=$baseDir/$book/$dataset
66			fi
67
68			# download the sample
69			echo " downloadSample.sh $line"
70	paus	1.1	downloadSample.sh $line
71
72	paus	1.3	# go into waiting loop
73	paus	1.1	nowTime=$(date +%s); duration=$(($nowTime - $startTime))
74	paus	1.6	jobs=`condor_q -global $USER -format "%d " ClusterId -format "%s " Cmd -format "%s \n" Args \| grep $dataset \| grep downloadFiles.sh \| wc -l`
75	paus	1.4	while [ $jobs -gt $remainingMax ]
76	paus	1.1	do
77			echo " waiting since $duration sec == condor queue has $jobs jobs left"; sleep 60; echo ""
78	paus	1.6	jobs=`condor_q -global $USER -format "%d " ClusterId -format "%s " Cmd -format "%s \n" Args \| grep $dataset \| grep downloadFiles.sh \| wc -l`
79	paus	1.1	nowTime=$(date +%s); duration=$(($nowTime - $startTime))
80			done
81			echo " Queues are empty ($jobs) --> cleaning up and making catalogs."
82			echo ""
83
84			# remove zero length files
85	paus	1.4	echo "removeZeroLengthFiles.sh $targetDir/$book/$dataset"
86			removeZeroLengthFiles.sh $targetDir/$book/$dataset
87	paus	1.1
88			# finally make the corresponding catalog
89	paus	1.4	echo "catalog.sh -ceg $version $dataset --retry $targetDir"
90			catalog.sh -ceg $version $dataset --retry $targetDir
91	paus	1.1
92			i=$(( $i+1 ))
93
94			done
95
96			exit 0