Processing/bin/downloadSample.sh

#!/bin/bash
#---------------------------------------------------------------------------------------------------
# Download a list of files
#---------------------------------------------------------------------------------------------------

# Read the arguments
echo ""
echo "downloadSample.sh  $*"
echo ""
dataDir=$1;      shift
book=$1;         shift
dataset=$1;      shift
target=$1;       shift
nCopyProcs=$1;   shift
condorOutput=$1; shift

# Prepare environment
echo " "
echo " Process dataset: $dataset  of book: $book"
echo "   in directory : $dataDir"
echo "   to target    : $target"
echo "   n copy procs : $nCopyProcs"
echo "   condor output: $condorOutput"

mkdir -p $condorOutput/$book/$dataset
mkdir -p $target/$book/$dataset
script=`which downloadFiles.sh`

# cleanup our lists
rm -f $condorOutput/$book/$dataset/fileList*.txt*

var=`echo $dataDir | grep /castor/cern.ch`
if [ "$var" != "" ]
then
  storageEle="srm-cms.cern.ch"
  storagePath='/srm/managerv2?SFN='
  storageUrl="srm://${storageEle}:8443${storagePath}$dataDir/$book/$dataset"
  #srmls $storageUrl | grep root | tr -s ' ' | cut -d' ' -f 2-3 2> /dev/null 1> \
  #  $condorOutput/$book/$dataset/fileList-all.txt-bak
  list $dataDir/$book/$dataset > $condorOutput/$book/$dataset/fileList-all.txt-bak
  stageSample.py --dataDir=$dataDir/$book/$dataset
else
  storageEle="se01.cmsaf.mit.edu"
  storagePath='/srm/managerv2?SFN='
  storageUrl="srm://${storageEle}:8443${storagePath}$dataDir/$book/$dataset"
  #echo "list $dataDir/$book/$dataset > $condorOutput/$book/$dataset/fileList-all.txt-bak"
  list $dataDir/$book/$dataset > $condorOutput/$book/$dataset/fileList-all.txt-bak
fi

# Make sure there is a kerberos ticket available
cp /tmp/x509up_u5410 ~/.krb5/
KRB5CCNAME=`klist -5 | grep 'Ticket cache:' | cut -d' ' -f 3`
if ! [ -z $KRB5CCNAME ]
then
  mkdir    -p  ~/.krb5/
  chmod 0      ~/.krb5
  chmod u=rwx  ~/.krb5
  file=`echo $KRB5CCNAME | cut -d: -f2`
  if [ -f "$file" ]
  then
    cp $file ~/.krb5/ticket
  else
    echo " ERROR -- missing kerberos ticket ($KRB5CCNAME)."
    exit 1
  fi
else
  echo " ERROR -- missing kerberos ticket ($KRB5CCNAME)."
fi

# make list of all remote files
#echo " converting all entries"
cat $condorOutput/$book/$dataset/fileList-all.txt-bak | grep root | \
while read line
do
  size=`echo $line | tr -s ' ' | cut -d ' ' -f 1`
  file=`echo $line | tr -s ' ' | cut -d ' ' -f 2`
  file=`basename $file`
  echo "$size $file" >> $condorOutput/$book/$dataset/fileList-all.txt
done

# make list of all local files
opt=''
if [ "`echo $HOSTNAME | grep mit.edu`" != "" ] && [ "`echo $dataDir | grep /castor/cern.ch`" != "" ]
then
  opt='--simple'
fi
#echo "list $opt $target/$book/$dataset | grep root > $condorOutput/$book/$dataset/fileList-done.txt"
list $opt $target/$book/$dataset | grep root > $condorOutput/$book/$dataset/fileList-done.txt

# make list of missing files
rm -f $condorOutput/$book/$dataset/fileList.txt
touch $condorOutput/$book/$dataset/fileList.txt
cat   $condorOutput/$book/$dataset/fileList-all.txt | grep root | \
while read line
do
  size=`echo $line | tr -s ' ' | cut -d ' ' -f 1`
  file=`echo $line | tr -s ' ' | cut -d ' ' -f 2`
  exists=`grep "$size $file" $condorOutput/$book/$dataset/fileList-done.txt`
  if [ "$exists" == "" ]
  then
    echo "   -missing- $file with $size bytes"
    echo "$size $file" >> $condorOutput/$book/$dataset/fileList.txt
  # else
  #   echo "   -exists-- $file with $size bytes - exists"
  fi
done

# construct our job
nFiles=`wc -l $condorOutput/$book/$dataset/fileList.txt | cut -d ' ' -f1`
if   [ "$nFiles" == "" ] || [ "$nFiles" == "0" ]
then
  echo " "
  echo " No more files to download. EXIT."
  exit 0
elif [ $nFiles -lt $nCopyProcs ]
then
  nCopyProcs=$nFiles
fi
nFilesPerJob=$(( $nFiles/$nCopyProcs ))
echo "   n files all  : $nFiles"
echo "   n files/proc : $nFilesPerJob"

i=1
next=1
last=$nFilesPerJob

while [ $i -le $nCopyProcs ]
do
  if [ $i == $nCopyProcs ]
  then
    last=$nFiles
  fi

  # say what we are going to submit
  echo "   downloadFiles.sh $dataDir $book $dataset $target $condorOutput $next $last"

  logFile=`echo download:$book/$dataset/${next}-${last}.txt | tr '/' '+'`
  logFile=/tmp/$logFile
  rm -f $logFile

  # prepare the condor_submit files
  cat > submit_$$.cmd <<EOF
Universe                = vanilla
Requirements            = ( (Arch == "X86_64" || Arch == "INTEL") && (OpSys == "LINUX") && (Disk >= DiskUsage) && ((Memory * 1024) >= ImageSize) && (HasFileTransfer) )
Notify_user             = paus@mit.edu
Notification            = Error
Executable              = $script
Arguments               = $dataDir $book $dataset $target $condorOutput $next $last
Rank                    = Mips
GetEnv                  = True
Input                   = /dev/null
Output                  = $condorOutput/$book/$dataset/${next}-${last}.out
Error                   = $condorOutput/$book/$dataset/${next}-${last}.err
Log                     = $logFile
should_transfer_files   = YES
when_to_transfer_output = ON_EXIT
Queue
EOF

  # submit the jobs
  condor_submit submit_$$.cmd >& /dev/null #>& lastSub
  #cat submit_$$.cmd
  rm  submit_$$.cmd

  # update counters
  next=$(( $next + $nFilesPerJob ))
  last=$(( $last + $nFilesPerJob ))
  i=$(( $i + 1 ))
done

exit 0
Revision:	1.2
Committed:	Fri Dec 10 09:41:33 2010 UTC (14 years, 5 months ago) by ceballos
Content type:	application/x-sh
Branch:	MAIN
Changes since 1.1:	+1 -0 lines
Log Message:	new code
#	User	Rev	Content
1	paus	1.1	#!/bin/bash
2			#---------------------------------------------------------------------------------------------------
3			# Download a list of files
4			#---------------------------------------------------------------------------------------------------
5
6			# Read the arguments
7			echo ""
8			echo "downloadSample.sh $*"
9			echo ""
10			dataDir=$1; shift
11			book=$1; shift
12			dataset=$1; shift
13			target=$1; shift
14			nCopyProcs=$1; shift
15			condorOutput=$1; shift
16
17			# Prepare environment
18			echo " "
19			echo " Process dataset: $dataset of book: $book"
20			echo " in directory : $dataDir"
21			echo " to target : $target"
22			echo " n copy procs : $nCopyProcs"
23			echo " condor output: $condorOutput"
24
25			mkdir -p $condorOutput/$book/$dataset
26			mkdir -p $target/$book/$dataset
27			script=`which downloadFiles.sh`
28
29			# cleanup our lists
30			rm -f $condorOutput/$book/$dataset/fileList.txt
31
32			var=`echo $dataDir \| grep /castor/cern.ch`
33			if [ "$var" != "" ]
34			then
35			storageEle="srm-cms.cern.ch"
36			storagePath='/srm/managerv2?SFN='
37			storageUrl="srm://${storageEle}:8443${storagePath}$dataDir/$book/$dataset"
38			#srmls $storageUrl \| grep root \| tr -s ' ' \| cut -d' ' -f 2-3 2> /dev/null 1> \
39			# $condorOutput/$book/$dataset/fileList-all.txt-bak
40			list $dataDir/$book/$dataset > $condorOutput/$book/$dataset/fileList-all.txt-bak
41	ceballos	1.2	stageSample.py --dataDir=$dataDir/$book/$dataset
42	paus	1.1	else
43			storageEle="se01.cmsaf.mit.edu"
44			storagePath='/srm/managerv2?SFN='
45			storageUrl="srm://${storageEle}:8443${storagePath}$dataDir/$book/$dataset"
46			#echo "list $dataDir/$book/$dataset > $condorOutput/$book/$dataset/fileList-all.txt-bak"
47			list $dataDir/$book/$dataset > $condorOutput/$book/$dataset/fileList-all.txt-bak
48			fi
49
50			# Make sure there is a kerberos ticket available
51			cp /tmp/x509up_u5410 ~/.krb5/
52			KRB5CCNAME=`klist -5 \| grep 'Ticket cache:' \| cut -d' ' -f 3`
53			if ! [ -z $KRB5CCNAME ]
54			then
55			mkdir -p ~/.krb5/
56			chmod 0 ~/.krb5
57			chmod u=rwx ~/.krb5
58			file=`echo $KRB5CCNAME \| cut -d: -f2`
59			if [ -f "$file" ]
60			then
61			cp $file ~/.krb5/ticket
62			else
63			echo " ERROR -- missing kerberos ticket ($KRB5CCNAME)."
64			exit 1
65			fi
66			else
67			echo " ERROR -- missing kerberos ticket ($KRB5CCNAME)."
68			fi
69
70			# make list of all remote files
71			#echo " converting all entries"
72			cat $condorOutput/$book/$dataset/fileList-all.txt-bak \| grep root \| \
73			while read line
74			do
75			size=`echo $line \| tr -s ' ' \| cut -d ' ' -f 1`
76			file=`echo $line \| tr -s ' ' \| cut -d ' ' -f 2`
77			file=`basename $file`
78			echo "$size $file" >> $condorOutput/$book/$dataset/fileList-all.txt
79			done
80
81			# make list of all local files
82			opt=''
83			if [ "`echo $HOSTNAME \| grep mit.edu`" != "" ] && [ "`echo $dataDir \| grep /castor/cern.ch`" != "" ]
84			then
85			opt='--simple'
86			fi
87			#echo "list $opt $target/$book/$dataset \| grep root > $condorOutput/$book/$dataset/fileList-done.txt"
88			list $opt $target/$book/$dataset \| grep root > $condorOutput/$book/$dataset/fileList-done.txt
89
90			# make list of missing files
91			rm -f $condorOutput/$book/$dataset/fileList.txt
92			touch $condorOutput/$book/$dataset/fileList.txt
93			cat $condorOutput/$book/$dataset/fileList-all.txt \| grep root \| \
94			while read line
95			do
96			size=`echo $line \| tr -s ' ' \| cut -d ' ' -f 1`
97			file=`echo $line \| tr -s ' ' \| cut -d ' ' -f 2`
98			exists=`grep "$size $file" $condorOutput/$book/$dataset/fileList-done.txt`
99			if [ "$exists" == "" ]
100			then
101			echo " -missing- $file with $size bytes"
102			echo "$size $file" >> $condorOutput/$book/$dataset/fileList.txt
103			# else
104			# echo " -exists-- $file with $size bytes - exists"
105			fi
106			done
107
108			# construct our job
109			nFiles=`wc -l $condorOutput/$book/$dataset/fileList.txt \| cut -d ' ' -f1`
110			if [ "$nFiles" == "" ] \|\| [ "$nFiles" == "0" ]
111			then
112			echo " "
113			echo " No more files to download. EXIT."
114			exit 0
115			elif [ $nFiles -lt $nCopyProcs ]
116			then
117			nCopyProcs=$nFiles
118			fi
119			nFilesPerJob=$(( $nFiles/$nCopyProcs ))
120			echo " n files all : $nFiles"
121			echo " n files/proc : $nFilesPerJob"
122
123			i=1
124			next=1
125			last=$nFilesPerJob
126
127			while [ $i -le $nCopyProcs ]
128			do
129			if [ $i == $nCopyProcs ]
130			then
131			last=$nFiles
132			fi
133
134			# say what we are going to submit
135			echo " downloadFiles.sh $dataDir $book $dataset $target $condorOutput $next $last"
136
137			logFile=`echo download:$book/$dataset/${next}-${last}.txt \| tr '/' '+'`
138			logFile=/tmp/$logFile
139			rm -f $logFile
140
141			# prepare the condor_submit files
142			cat > submit_$$.cmd <<EOF
143			Universe = vanilla
144			Requirements = ( (Arch == "X86_64" \|\| Arch == "INTEL") && (OpSys == "LINUX") && (Disk >= DiskUsage) && ((Memory * 1024) >= ImageSize) && (HasFileTransfer) )
145			Notify_user = paus@mit.edu
146			Notification = Error
147			Executable = $script
148			Arguments = $dataDir $book $dataset $target $condorOutput $next $last
149			Rank = Mips
150			GetEnv = True
151			Input = /dev/null
152			Output = $condorOutput/$book/$dataset/${next}-${last}.out
153			Error = $condorOutput/$book/$dataset/${next}-${last}.err
154			Log = $logFile
155			should_transfer_files = YES
156			when_to_transfer_output = ON_EXIT
157			Queue
158			EOF
159
160			# submit the jobs
161			condor_submit submit_$$.cmd >& /dev/null #>& lastSub
162			#cat submit_$$.cmd
163			rm submit_$$.cmd
164
165			# update counters
166			next=$(( $next + $nFilesPerJob ))
167			last=$(( $last + $nFilesPerJob ))
168			i=$(( $i + 1 ))
169			done
170
171			exit 0