Processing/bin/download.sh

#!/bin/bash
#---------------------------------------------------------------------------------------------------
# Go through a number of download requests
#---------------------------------------------------------------------------------------------------
H=`basename $0`
SAMPLE_LIST=$1

# start next request
remainingMax=0

# health checks
if [ ".$1" == "." ]
then
  echo "";
  echo "usage: $H  <sampleList>";
  echo ""
  exit 1
fi

# say what we do
echo ""
echo " ==== Download queue from $SAMPLE_LIST ===="
echo ""

# check that queues are really empty
dataset="XXX"
startTime=$(date +%s)
nowTime=$(date +%s); duration=$(($nowTime - $startTime))
jobs=`condor_q -global $USER -format "%d " ClusterId -format "%s " Cmd -format "%s \n" Args | grep $dataset | grep downloadFiles.sh | wc -l`
while [ $jobs -gt $remainingMax ]
do
  echo " waiting since  $duration sec  == condor queue has  $jobs jobs  left"; sleep 60; echo ""
  jobs=`condor_q -global $USER $format | grep $dataset | grep downloadFiles.sh | wc -l`
  #jobs=`condor_q -global $USER | grep downloadFiles.sh | wc -l`
  nowTime=$(date +%s); duration=$(($nowTime - $startTime))
done
echo " Queues are close to empty ($jobs) -- Let's get started."
echo ""

# all right, ready to download
nSamples=`cat $SAMPLE_LIST | wc -l | cut -d ' ' -f1`
i=1
while [ $i -le $nSamples ]
do
  startTime=$(date +%s)

  line=`head -$i $SAMPLE_LIST | tail -1`
  if [ "`echo $line | grep ^#`" != "" ]
  then
    i=$(( $i+1 ))
    continue
  fi

  # define some quantities
  baseDir=`echo $line | tr -s ' ' | cut -d ' ' -f 1`
  book=`echo $line | tr -s ' ' | cut -d ' ' -f 2`
  version=`basename $book`
  dataset=`echo $line | tr -s ' ' | cut -d ' ' -f 3`
  targetDir=`echo $line | tr -s ' ' | cut -d ' ' -f 4`

  
  # stagein the sample if it is at CERN
  if [ "`echo $baseDir | grep /castor/cern.ch`" != "" ]
  then
    echo "  ssh paus@lxplus.cern.ch ./stageSample.py --dataDir=$baseDir/$book/$dataset"
    ssh paus@lxplus.cern.ch ./stageSample.py --dataDir=$baseDir/$book/$dataset
  fi
  
  # download the sample
  echo " downloadSample.sh $line"
  downloadSample.sh $line
  
  # go into waiting loop
  nowTime=$(date +%s); duration=$(($nowTime - $startTime))
  jobs=`condor_q -global $USER -format "%d " ClusterId -format "%s " Cmd -format "%s \n" Args | grep $dataset | grep downloadFiles.sh | wc -l`
  while [ $jobs -gt $remainingMax ]
  do
    echo " waiting since  $duration sec  == condor queue has  $jobs jobs  left"; sleep 60; echo ""
    jobs=`condor_q -global $USER -format "%d " ClusterId -format "%s " Cmd -format "%s \n" Args | grep $dataset | grep downloadFiles.sh | wc -l`
    nowTime=$(date +%s); duration=$(($nowTime - $startTime))
  done
  echo " Queues are empty ($jobs) --> cleaning up and making catalogs."
  echo ""
  
  # remove zero length files
  echo "removeZeroLengthFiles.sh $targetDir/$book/$dataset"
  removeZeroLengthFiles.sh $targetDir/$book/$dataset
  
  # finally make the corresponding catalog
  if [ ${#version} == 3 ]
  then
    echo "catalog.sh -ceg $version $dataset --retry $targetDir"
    catalog.sh -ceg $version $dataset --retry $targetDir
  else
    echo "This is not a normal dataset, make a simple catalog."
    echo ""
    list $targetDir/$book/$dataset > /tmp/list.$$
    mkdir -p ~/catalog/t2mit/private/$book/$dataset
    rm -f    ~/catalog/t2mit/private/$book/$dataset/Files
    touch    ~/catalog/t2mit/private/$book/$dataset/Files
    while read line
    do
      size=`echo $line | cut -d ' ' -f1`
      file=`echo $line | cut -d ' ' -f2`
      root -l -b -q $MIT_PROD_DIR/root/runSimpleFileCataloger.C\(\"$targetDir/$book/$dataset\",\"$file\"\) \
          2> /dev/null | grep CATALOG >> ~/catalog/t2mit/private/$book/$dataset/Files
    done < /tmp/list.$$
    rm -f /tmp/list.$$
  fi

  i=$(( $i+1 ))

done

exit 0
Revision:	1.8
Committed:	Sat Oct 22 13:07:04 2011 UTC (13 years, 6 months ago) by paus
Content type:	application/x-sh
Branch:	MAIN
CVS Tags:	Mit_025c_branch1, Mit_025c_branch0, Mit_025c, Mit_025b, Mit_025a
Branch point for:	Mit_025c_branch
Changes since 1.7:	+22 -3 lines
Log Message:	Private file productions.
#	User	Rev	Content
1	paus	1.1	#!/bin/bash
2			#---------------------------------------------------------------------------------------------------
3			# Go through a number of download requests
4			#---------------------------------------------------------------------------------------------------
5			H=`basename $0`
6			SAMPLE_LIST=$1
7
8	paus	1.4	# start next request
9			remainingMax=0
10
11	paus	1.1	# health checks
12			if [ ".$1" == "." ]
13			then
14			echo "";
15			echo "usage: $H <sampleList>";
16			echo ""
17			exit 1
18			fi
19
20			# say what we do
21			echo ""
22			echo " ==== Download queue from $SAMPLE_LIST ===="
23			echo ""
24
25			# check that queues are really empty
26	paus	1.4	dataset="XXX"
27	paus	1.1	startTime=$(date +%s)
28			nowTime=$(date +%s); duration=$(($nowTime - $startTime))
29	paus	1.6	jobs=`condor_q -global $USER -format "%d " ClusterId -format "%s " Cmd -format "%s \n" Args \| grep $dataset \| grep downloadFiles.sh \| wc -l`
30	paus	1.4	while [ $jobs -gt $remainingMax ]
31	paus	1.1	do
32			echo " waiting since $duration sec == condor queue has $jobs jobs left"; sleep 60; echo ""
33	paus	1.5	jobs=`condor_q -global $USER $format \| grep $dataset \| grep downloadFiles.sh \| wc -l`
34	paus	1.4	#jobs=`condor_q -global $USER \| grep downloadFiles.sh \| wc -l`
35	paus	1.1	nowTime=$(date +%s); duration=$(($nowTime - $startTime))
36			done
37	paus	1.3	echo " Queues are close to empty ($jobs) -- Let's get started."
38	paus	1.1	echo ""
39
40			# all right, ready to download
41			nSamples=`cat $SAMPLE_LIST \| wc -l \| cut -d ' ' -f1`
42			i=1
43			while [ $i -le $nSamples ]
44			do
45	paus	1.5	startTime=$(date +%s)
46	paus	1.1
47			line=`head -$i $SAMPLE_LIST \| tail -1`
48			if [ "`echo $line \| grep ^#`" != "" ]
49			then
50			i=$(( $i+1 ))
51			continue
52			fi
53
54			# define some quantities
55	paus	1.3	baseDir=`echo $line \| tr -s ' ' \| cut -d ' ' -f 1`
56	paus	1.1	book=`echo $line \| tr -s ' ' \| cut -d ' ' -f 2`
57			version=`basename $book`
58			dataset=`echo $line \| tr -s ' ' \| cut -d ' ' -f 3`
59	paus	1.4	targetDir=`echo $line \| tr -s ' ' \| cut -d ' ' -f 4`
60	paus	1.1
61	paus	1.8
62	paus	1.3	# stagein the sample if it is at CERN
63			if [ "`echo $baseDir \| grep /castor/cern.ch`" != "" ]
64			then
65	paus	1.4	echo " ssh paus@lxplus.cern.ch ./stageSample.py --dataDir=$baseDir/$book/$dataset"
66	paus	1.3	ssh paus@lxplus.cern.ch ./stageSample.py --dataDir=$baseDir/$book/$dataset
67			fi
68	paus	1.8
69	paus	1.3	# download the sample
70			echo " downloadSample.sh $line"
71	paus	1.1	downloadSample.sh $line
72
73	paus	1.3	# go into waiting loop
74	paus	1.1	nowTime=$(date +%s); duration=$(($nowTime - $startTime))
75	paus	1.6	jobs=`condor_q -global $USER -format "%d " ClusterId -format "%s " Cmd -format "%s \n" Args \| grep $dataset \| grep downloadFiles.sh \| wc -l`
76	paus	1.4	while [ $jobs -gt $remainingMax ]
77	paus	1.1	do
78			echo " waiting since $duration sec == condor queue has $jobs jobs left"; sleep 60; echo ""
79	paus	1.6	jobs=`condor_q -global $USER -format "%d " ClusterId -format "%s " Cmd -format "%s \n" Args \| grep $dataset \| grep downloadFiles.sh \| wc -l`
80	paus	1.1	nowTime=$(date +%s); duration=$(($nowTime - $startTime))
81			done
82			echo " Queues are empty ($jobs) --> cleaning up and making catalogs."
83			echo ""
84
85			# remove zero length files
86	paus	1.4	echo "removeZeroLengthFiles.sh $targetDir/$book/$dataset"
87			removeZeroLengthFiles.sh $targetDir/$book/$dataset
88	paus	1.1
89			# finally make the corresponding catalog
90	paus	1.8	if [ ${#version} == 3 ]
91			then
92			echo "catalog.sh -ceg $version $dataset --retry $targetDir"
93			catalog.sh -ceg $version $dataset --retry $targetDir
94			else
95			echo "This is not a normal dataset, make a simple catalog."
96			echo ""
97			list $targetDir/$book/$dataset > /tmp/list.$$
98			mkdir -p ~/catalog/t2mit/private/$book/$dataset
99			rm -f ~/catalog/t2mit/private/$book/$dataset/Files
100			touch ~/catalog/t2mit/private/$book/$dataset/Files
101			while read line
102			do
103			size=`echo $line \| cut -d ' ' -f1`
104			file=`echo $line \| cut -d ' ' -f2`
105			root -l -b -q $MIT_PROD_DIR/root/runSimpleFileCataloger.C\(\"$targetDir/$book/$dataset\",\"$file\"\) \
106			2> /dev/null \| grep CATALOG >> ~/catalog/t2mit/private/$book/$dataset/Files
107			done < /tmp/list.$$
108			rm -f /tmp/list.$$
109			fi
110	paus	1.1
111			i=$(( $i+1 ))
112
113			done
114
115			exit 0