Processing/bin/download.sh

#!/bin/bash
#---------------------------------------------------------------------------------------------------
# Go through a number of download requests
#---------------------------------------------------------------------------------------------------
H=`basename $0`
SAMPLE_LIST=$1

# start next request
remainingMax=0

# health checks
if [ ".$1" == "." ]
then
  echo "";
  echo "usage: $H  <sampleList>";
  echo ""
  exit 1
fi

# say what we do
echo ""
echo " ==== Download queue from $SAMPLE_LIST ===="
echo ""

# check that queues are really empty
dataset="XXX"
startTime=$(date +%s)
nowTime=$(date +%s); duration=$(($nowTime - $startTime))
jobs=`condor_q -global $USER -format "%s " Cmd -format "%s \n" Args | grep $dataset | grep downloadFiles.sh | wc -l`
while [ $jobs -gt $remainingMax ]
do
  echo " waiting since  $duration sec  == condor queue has  $jobs jobs  left"; sleep 60; echo ""
  jobs=`condor_q -global $USER -format "%s " Cmd -format "%s \n" Args | grep $dataset | grep downloadFiles.sh | wc -l`
  nowTime=$(date +%s); duration=$(($nowTime - $startTime))
done
echo " Queues are close to empty ($jobs) -- Let's get started."
echo ""

# all right, ready to download
nSamples=`cat $SAMPLE_LIST | wc -l | cut -d ' ' -f1`
i=1
while [ $i -le $nSamples ]
do
  startTime=$(date +%s)

  line=`head -$i $SAMPLE_LIST | tail -1`
  if [ "`echo $line | grep ^#`" != "" ]
  then
    i=$(( $i+1 ))
    continue
  fi

  # define some quantities
  baseDir=`echo $line | tr -s ' ' | cut -d ' ' -f 1`
  book=`echo $line | tr -s ' ' | cut -d ' ' -f 2`
  version=`basename $book`
  dataset=`echo $line | tr -s ' ' | cut -d ' ' -f 3`
  targetDir=`echo $line | tr -s ' ' | cut -d ' ' -f 4`

  
  # stagein the sample if it is at CERN
  if [ "`echo $baseDir | grep /castor/cern.ch`" != "" ]
  then
    echo "  ssh -x paus@lxplus.cern.ch ./stageSample.py --dataDir=$baseDir/$book/$dataset"
    ssh -x paus@lxplus.cern.ch ./stageSample.py --dataDir=$baseDir/$book/$dataset
  fi
  
  # download the sample
  downloadSample.sh $line
  
  # go into waiting loop
  nowTime=$(date +%s); duration=$(($nowTime - $startTime))
  jobs=`condor_q -global $USER -format "%s " Cmd -format "%s \n" Args | grep $dataset | grep downloadFiles.sh | wc -l`
  while [ $jobs -gt $remainingMax ]
  do
    echo " waiting since  $duration sec  == condor queue has  $jobs jobs  left"; sleep 60; echo ""
    jobs=`condor_q -global $USER -format "%s " Cmd -format "%s \n" Args | grep $dataset | grep downloadFiles.sh | wc -l`
    nowTime=$(date +%s); duration=$(($nowTime - $startTime))
  done
  echo " Queues are empty ($jobs) --> cleaning up and making catalogs."
  echo ""
  
  # remove zero length files
  echo "removeZeroLengthFiles.sh $targetDir/$book/$dataset"
  removeZeroLengthFiles.sh $targetDir/$book/$dataset
  
  # finally make the corresponding catalog
  if [ ${#version} == 3 ]
  then
    echo "catalog.sh -ceg $version $dataset --retry $targetDir"
    catalog.sh -ceg $version $dataset --retry $targetDir
  else
    echo "This is not a normal dataset, make a simple catalog."
    echo ""
    list $targetDir/$book/$dataset > /tmp/list.$$
    mkdir -p ~/catalog/t2mit/private/$book/$dataset
    rm -f    ~/catalog/t2mit/private/$book/$dataset/Files
    touch    ~/catalog/t2mit/private/$book/$dataset/Files
    while read line
    do
      size=`echo $line | cut -d ' ' -f1`
      file=`echo $line | cut -d ' ' -f2`
      root -l -b -q $MIT_PROD_DIR/root/runSimpleFileCataloger.C\(\"$targetDir/$book/$dataset\",\"$file\"\) \
          2> /dev/null | grep CATALOG >> ~/catalog/t2mit/private/$book/$dataset/Files
    done < /tmp/list.$$
    rm -f /tmp/list.$$
  fi

  i=$(( $i+1 ))

done

exit 0
Revision:	1.9
Committed:	Tue Feb 28 11:54:35 2012 UTC (13 years, 2 months ago) by paus
Content type:	application/x-sh
Branch:	MAIN
CVS Tags:	Mit_028a, Mit_028, Mit_027a, Mit_027, Mit_026, Mit_025e, Mit_025d
Changes since 1.8:	+6 -8 lines
Log Message:	Last updates.
#	User	Rev	Content
1	paus	1.1	#!/bin/bash
2			#---------------------------------------------------------------------------------------------------
3			# Go through a number of download requests
4			#---------------------------------------------------------------------------------------------------
5			H=`basename $0`
6			SAMPLE_LIST=$1
7
8	paus	1.4	# start next request
9			remainingMax=0
10
11	paus	1.1	# health checks
12			if [ ".$1" == "." ]
13			then
14			echo "";
15			echo "usage: $H <sampleList>";
16			echo ""
17			exit 1
18			fi
19
20			# say what we do
21			echo ""
22			echo " ==== Download queue from $SAMPLE_LIST ===="
23			echo ""
24
25			# check that queues are really empty
26	paus	1.4	dataset="XXX"
27	paus	1.1	startTime=$(date +%s)
28			nowTime=$(date +%s); duration=$(($nowTime - $startTime))
29	paus	1.9	jobs=`condor_q -global $USER -format "%s " Cmd -format "%s \n" Args \| grep $dataset \| grep downloadFiles.sh \| wc -l`
30	paus	1.4	while [ $jobs -gt $remainingMax ]
31	paus	1.1	do
32			echo " waiting since $duration sec == condor queue has $jobs jobs left"; sleep 60; echo ""
33	paus	1.9	jobs=`condor_q -global $USER -format "%s " Cmd -format "%s \n" Args \| grep $dataset \| grep downloadFiles.sh \| wc -l`
34	paus	1.1	nowTime=$(date +%s); duration=$(($nowTime - $startTime))
35			done
36	paus	1.3	echo " Queues are close to empty ($jobs) -- Let's get started."
37	paus	1.1	echo ""
38
39			# all right, ready to download
40			nSamples=`cat $SAMPLE_LIST \| wc -l \| cut -d ' ' -f1`
41			i=1
42			while [ $i -le $nSamples ]
43			do
44	paus	1.5	startTime=$(date +%s)
45	paus	1.1
46			line=`head -$i $SAMPLE_LIST \| tail -1`
47			if [ "`echo $line \| grep ^#`" != "" ]
48			then
49			i=$(( $i+1 ))
50			continue
51			fi
52
53			# define some quantities
54	paus	1.3	baseDir=`echo $line \| tr -s ' ' \| cut -d ' ' -f 1`
55	paus	1.1	book=`echo $line \| tr -s ' ' \| cut -d ' ' -f 2`
56			version=`basename $book`
57			dataset=`echo $line \| tr -s ' ' \| cut -d ' ' -f 3`
58	paus	1.4	targetDir=`echo $line \| tr -s ' ' \| cut -d ' ' -f 4`
59	paus	1.1
60	paus	1.8
61	paus	1.3	# stagein the sample if it is at CERN
62			if [ "`echo $baseDir \| grep /castor/cern.ch`" != "" ]
63			then
64	paus	1.9	echo " ssh -x paus@lxplus.cern.ch ./stageSample.py --dataDir=$baseDir/$book/$dataset"
65			ssh -x paus@lxplus.cern.ch ./stageSample.py --dataDir=$baseDir/$book/$dataset
66	paus	1.3	fi
67	paus	1.8
68	paus	1.3	# download the sample
69	paus	1.1	downloadSample.sh $line
70
71	paus	1.3	# go into waiting loop
72	paus	1.1	nowTime=$(date +%s); duration=$(($nowTime - $startTime))
73	paus	1.9	jobs=`condor_q -global $USER -format "%s " Cmd -format "%s \n" Args \| grep $dataset \| grep downloadFiles.sh \| wc -l`
74	paus	1.4	while [ $jobs -gt $remainingMax ]
75	paus	1.1	do
76			echo " waiting since $duration sec == condor queue has $jobs jobs left"; sleep 60; echo ""
77	paus	1.9	jobs=`condor_q -global $USER -format "%s " Cmd -format "%s \n" Args \| grep $dataset \| grep downloadFiles.sh \| wc -l`
78	paus	1.1	nowTime=$(date +%s); duration=$(($nowTime - $startTime))
79			done
80			echo " Queues are empty ($jobs) --> cleaning up and making catalogs."
81			echo ""
82
83			# remove zero length files
84	paus	1.4	echo "removeZeroLengthFiles.sh $targetDir/$book/$dataset"
85			removeZeroLengthFiles.sh $targetDir/$book/$dataset
86	paus	1.1
87			# finally make the corresponding catalog
88	paus	1.8	if [ ${#version} == 3 ]
89			then
90			echo "catalog.sh -ceg $version $dataset --retry $targetDir"
91			catalog.sh -ceg $version $dataset --retry $targetDir
92			else
93			echo "This is not a normal dataset, make a simple catalog."
94			echo ""
95			list $targetDir/$book/$dataset > /tmp/list.$$
96			mkdir -p ~/catalog/t2mit/private/$book/$dataset
97			rm -f ~/catalog/t2mit/private/$book/$dataset/Files
98			touch ~/catalog/t2mit/private/$book/$dataset/Files
99			while read line
100			do
101			size=`echo $line \| cut -d ' ' -f1`
102			file=`echo $line \| cut -d ' ' -f2`
103			root -l -b -q $MIT_PROD_DIR/root/runSimpleFileCataloger.C\(\"$targetDir/$book/$dataset\",\"$file\"\) \
104			2> /dev/null \| grep CATALOG >> ~/catalog/t2mit/private/$book/$dataset/Files
105			done < /tmp/list.$$
106			rm -f /tmp/list.$$
107			fi
108	paus	1.1
109			i=$(( $i+1 ))
110
111			done
112
113			exit 0