Processing/bin/downloadSample.sh

#!/bin/bash
#---------------------------------------------------------------------------------------------------
# Download a list of files
#---------------------------------------------------------------------------------------------------

# Read the arguments
echo ""
echo "downloadSample.sh  $*"
echo ""
dataDir=$1;      shift
book=$1;         shift
dataset=$1;      shift
target=$1;       shift
nCopyProcs=$1;   shift
condorOutput=$1; shift
onlyMissing=$1;  shift

DN=`grid-proxy-info -subject`

# Prepare environment
echo " "
echo " Process dataset: $dataset  of book: $book"
echo "   in directory : $dataDir"
echo "   to target    : $target"
echo "   n copy procs : $nCopyProcs"
echo "   condor output: $condorOutput"
echo "   only missing : $onlyMissing"

mkdir -p $condorOutput/$book/$dataset
makedir --exe  $target/$book/$dataset
script=`which downloadFiles.sh`

# cleanup our lists and remake a clean one
#echo "rm -f $condorOutput/$book/$dataset/fileList*.txt*"
rm -f $condorOutput/$book/$dataset/fileList*.txt*

# make list of all local files
if [ "`echo $HOSTNAME | grep mit.edu`" != "" ] && \
 ( [ "`echo $dataDir | grep /castor/cern.ch`" != "" ] || \
   [ "`echo $target | grep /castor/cern.ch`" != "" ] )
then
  opt="--simple"
else
  opt=""
fi

#echo "list $opt $dataDir/$book/$dataset > $condorOutput/$book/$dataset/fileList-all.txt-bak"
list $opt $dataDir/$book/$dataset > $condorOutput/$book/$dataset/fileList-all.txt-bak

# Make sure there is a kerberos ticket available
id=`id -u`
cp /tmp/x509up_u${id} ~/.krb5/
KRB5CCNAME=`klist -5 | grep 'Ticket cache:' | cut -d' ' -f 3`
if ! [ -z $KRB5CCNAME ]
then
  mkdir    -p  ~/.krb5/
  chmod 0      ~/.krb5
  chmod u=rwx  ~/.krb5
  file=`echo $KRB5CCNAME | cut -d: -f2`
  if [ -f "$file" ]
  then
    cp $file ~/.krb5/ticket
  else
    echo " ERROR -- missing kerberos ticket ($KRB5CCNAME)."
    exit 1
  fi
else
  echo " ERROR -- missing kerberos ticket ($KRB5CCNAME)."
fi

# make list of all remote files
#echo " converting all entries"
cat $condorOutput/$book/$dataset/fileList-all.txt-bak | grep root | \
while read line
do
  size=`echo $line | tr -s ' ' | cut -d ' ' -f 1`
  file=`echo $line | tr -s ' ' | cut -d ' ' -f 2`
  file=`basename $file`
  echo "$size $file" >> $condorOutput/$book/$dataset/fileList-all.txt
done

# make list of all local files
if [ "`echo $HOSTNAME | grep mit.edu`" != "" ] && ( [ "`echo $dataDir | grep /castor/cern.ch`" != "" ] || [ "`echo $target | grep /castor/cern.ch`" != "" ] )
then
  opt="--simple"
else
  opt=""
fi

#echo "list $opt $target/$book/$dataset | grep root > $condorOutput/$book/$dataset/fileList-done.txt"
list $opt $target/$book/$dataset | grep root > $condorOutput/$book/$dataset/fileList-done.txt

# make list of missing files
rm -f $condorOutput/$book/$dataset/fileList.txt
touch $condorOutput/$book/$dataset/fileList.txt
cat   $condorOutput/$book/$dataset/fileList-all.txt | grep root | \
while read line
do
  size=`echo $line | tr -s ' ' | cut -d ' ' -f 1`
  file=`echo $line | tr -s ' ' | cut -d ' ' -f 2`
  exists=`grep "$file" $condorOutput/$book/$dataset/fileList-done.txt`
  if [ "$exists" == "" ]
  then
    echo "   -missing-- $file with $size bytes"
    echo "$size $file" >> $condorOutput/$book/$dataset/fileList.txt
  # else
  #   echo "   -exists--- $file with $size bytes - exists"
  else
    # now check that size matches
    test=`grep "$size $file" $condorOutput/$book/$dataset/fileList-done.txt`
    if [ "$test" == "" ]
    then
      if [ "$onlyMissing" == "" ]
      then
        echo "   -fileSize- $exists (remote: $size)"
        echo "$size $file" >> $condorOutput/$book/$dataset/fileList.txt
      fi
    fi
  fi
done

# construct our job
nFiles=`wc -l $condorOutput/$book/$dataset/fileList.txt | cut -d ' ' -f1`
if   [ "$nFiles" == "" ] || [ "$nFiles" == "0" ]
then
  echo " "
  echo " No more files to download. EXIT."
  exit 0
elif [ $nFiles -lt $nCopyProcs ]
then
  nCopyProcs=$nFiles
fi
# how many files per job?
nFilesPerJob=$(( $nFiles/$nCopyProcs ))
nFilesTmp=$(( $nFilesPerJob*$nCopyProcs ))
if [ $nFilesPerJob == 1 ] && [ $nFiles -gt $nCopyProcs ]
then
  nFilesPerJob=2
elif [ $nFilesTmp -lt $nFiles ]
then
  nFilesPerJob=$(( $nFilesPerJob+1 ))
fi

echo "   n files all  : $nFiles"
echo "   n files/proc : $nFilesPerJob"

i=1
next=1
last=$nFilesPerJob

# make sure condor is properly setup for us
if ! [ -z $CONDOR_LOCATION ]
then
  unset  CONDOR_LOCATION
  export CONDOR_CONFIG=/usr/local/condor/etc/condor_config
fi

# loop over the condor jobs and submit them
while [ $i -le $nCopyProcs ] && [ $last -le $nFiles ]
do
  if [ $i == $nCopyProcs ]
  then
    last=$nFiles
  fi

  # say what we are going to submit
  echo "   downloadFiles.sh $dataDir $book $dataset $target $condorOutput $next $last"

  logFile=`echo download:$book/$dataset/${next}-${last}.txt | tr '/' '+'`
  logFile=/tmp/$logFile
  rm -f $logFile

  # prepare the condor_submit files
  cat > submit_$$.cmd <<EOF
Universe                = vanilla
Requirements            = ( (Arch == "X86_64") && (OpSys == "LINUX" || Arch == "INTEL") && (Disk >= DiskUsage) && ((Memory * 1024) >= ImageSize) && (HasFileTransfer) )
Notify_user             = paus@mit.edu
Notification            = Error
Executable              = $script
Arguments               = $dataDir $book $dataset $target $condorOutput $next $last
Rank                    = Mips
GetEnv                  = True
Input                   = /dev/null
Output                  = $condorOutput/$book/$dataset/${next}-${last}.out
Error                   = $condorOutput/$book/$dataset/${next}-${last}.err
Log                     = $logFile
should_transfer_files   = YES
when_to_transfer_output = ON_EXIT

+AccountingGroup        = "group_cmsuser.cmsu0284"

Queue
EOF

#+x509userproxysubject   = $DN

  # submit the jobs
  condor_submit submit_$$.cmd >& /dev/null #>& lastSub
  #cat submit_$$.cmd
  rm  submit_$$.cmd

  # update counters
  next=$(( $next + $nFilesPerJob ))
  last=$(( $last + $nFilesPerJob ))
  i=$(( $i + 1 ))
done

exit 0
Revision:	1.6
Committed:	Thu Mar 31 08:37:21 2011 UTC (14 years, 1 month ago) by ceballos
Content type:	application/x-sh
Branch:	MAIN
CVS Tags:	Mit_023, Mit_022a, Mit_022, Mit_020d, TMit_020d, Mit_020c, Mit_021, Mit_021pre2, Mit_021pre1, Mit_020b
Changes since 1.5:	+1 -1 lines
Log Message:	fix
#	User	Rev	Content
1	paus	1.1	#!/bin/bash
2			#---------------------------------------------------------------------------------------------------
3			# Download a list of files
4			#---------------------------------------------------------------------------------------------------
5
6			# Read the arguments
7			echo ""
8			echo "downloadSample.sh $*"
9			echo ""
10			dataDir=$1; shift
11			book=$1; shift
12			dataset=$1; shift
13			target=$1; shift
14			nCopyProcs=$1; shift
15			condorOutput=$1; shift
16	paus	1.3	onlyMissing=$1; shift
17
18			DN=`grid-proxy-info -subject`
19	paus	1.1
20			# Prepare environment
21			echo " "
22			echo " Process dataset: $dataset of book: $book"
23			echo " in directory : $dataDir"
24			echo " to target : $target"
25			echo " n copy procs : $nCopyProcs"
26			echo " condor output: $condorOutput"
27	paus	1.3	echo " only missing : $onlyMissing"
28	paus	1.1
29			mkdir -p $condorOutput/$book/$dataset
30	paus	1.4	makedir --exe $target/$book/$dataset
31	paus	1.1	script=`which downloadFiles.sh`
32
33	paus	1.3	# cleanup our lists and remake a clean one
34	paus	1.4	#echo "rm -f $condorOutput/$book/$dataset/fileList.txt"
35	paus	1.1	rm -f $condorOutput/$book/$dataset/fileList.txt
36
37	paus	1.3	# make list of all local files
38	paus	1.4	if [ "`echo $HOSTNAME \| grep mit.edu`" != "" ] && \
39			( [ "`echo $dataDir \| grep /castor/cern.ch`" != "" ] \|\| \
40			[ "`echo $target \| grep /castor/cern.ch`" != "" ] )
41	paus	1.1	then
42	paus	1.3	opt="--simple"
43	paus	1.1	else
44	paus	1.3	opt=""
45	paus	1.1	fi
46
47	paus	1.3	#echo "list $opt $dataDir/$book/$dataset > $condorOutput/$book/$dataset/fileList-all.txt-bak"
48			list $opt $dataDir/$book/$dataset > $condorOutput/$book/$dataset/fileList-all.txt-bak
49
50	paus	1.1	# Make sure there is a kerberos ticket available
51	paus	1.3	id=`id -u`
52			cp /tmp/x509up_u${id} ~/.krb5/
53	paus	1.1	KRB5CCNAME=`klist -5 \| grep 'Ticket cache:' \| cut -d' ' -f 3`
54			if ! [ -z $KRB5CCNAME ]
55			then
56			mkdir -p ~/.krb5/
57			chmod 0 ~/.krb5
58			chmod u=rwx ~/.krb5
59			file=`echo $KRB5CCNAME \| cut -d: -f2`
60			if [ -f "$file" ]
61			then
62			cp $file ~/.krb5/ticket
63			else
64			echo " ERROR -- missing kerberos ticket ($KRB5CCNAME)."
65			exit 1
66			fi
67			else
68			echo " ERROR -- missing kerberos ticket ($KRB5CCNAME)."
69			fi
70
71			# make list of all remote files
72			#echo " converting all entries"
73			cat $condorOutput/$book/$dataset/fileList-all.txt-bak \| grep root \| \
74			while read line
75			do
76			size=`echo $line \| tr -s ' ' \| cut -d ' ' -f 1`
77			file=`echo $line \| tr -s ' ' \| cut -d ' ' -f 2`
78			file=`basename $file`
79			echo "$size $file" >> $condorOutput/$book/$dataset/fileList-all.txt
80			done
81
82			# make list of all local files
83	paus	1.3	if [ "`echo $HOSTNAME \| grep mit.edu`" != "" ] && ( [ "`echo $dataDir \| grep /castor/cern.ch`" != "" ] \|\| [ "`echo $target \| grep /castor/cern.ch`" != "" ] )
84	paus	1.1	then
85	paus	1.3	opt="--simple"
86			else
87			opt=""
88	paus	1.1	fi
89	paus	1.3
90	paus	1.1	#echo "list $opt $target/$book/$dataset \| grep root > $condorOutput/$book/$dataset/fileList-done.txt"
91			list $opt $target/$book/$dataset \| grep root > $condorOutput/$book/$dataset/fileList-done.txt
92
93			# make list of missing files
94			rm -f $condorOutput/$book/$dataset/fileList.txt
95			touch $condorOutput/$book/$dataset/fileList.txt
96			cat $condorOutput/$book/$dataset/fileList-all.txt \| grep root \| \
97			while read line
98			do
99			size=`echo $line \| tr -s ' ' \| cut -d ' ' -f 1`
100			file=`echo $line \| tr -s ' ' \| cut -d ' ' -f 2`
101	paus	1.3	exists=`grep "$file" $condorOutput/$book/$dataset/fileList-done.txt`
102	paus	1.1	if [ "$exists" == "" ]
103			then
104	paus	1.3	echo " -missing-- $file with $size bytes"
105	paus	1.1	echo "$size $file" >> $condorOutput/$book/$dataset/fileList.txt
106			# else
107	paus	1.3	# echo " -exists--- $file with $size bytes - exists"
108			else
109			# now check that size matches
110			test=`grep "$size $file" $condorOutput/$book/$dataset/fileList-done.txt`
111			if [ "$test" == "" ]
112			then
113			if [ "$onlyMissing" == "" ]
114			then
115			echo " -fileSize- $exists (remote: $size)"
116			echo "$size $file" >> $condorOutput/$book/$dataset/fileList.txt
117			fi
118			fi
119	paus	1.1	fi
120			done
121
122			# construct our job
123			nFiles=`wc -l $condorOutput/$book/$dataset/fileList.txt \| cut -d ' ' -f1`
124			if [ "$nFiles" == "" ] \|\| [ "$nFiles" == "0" ]
125			then
126			echo " "
127			echo " No more files to download. EXIT."
128			exit 0
129			elif [ $nFiles -lt $nCopyProcs ]
130			then
131			nCopyProcs=$nFiles
132			fi
133	paus	1.4	# how many files per job?
134	paus	1.1	nFilesPerJob=$(( $nFiles/$nCopyProcs ))
135	paus	1.4	nFilesTmp=$(( $nFilesPerJob*$nCopyProcs ))
136			if [ $nFilesPerJob == 1 ] && [ $nFiles -gt $nCopyProcs ]
137			then
138			nFilesPerJob=2
139			elif [ $nFilesTmp -lt $nFiles ]
140			then
141			nFilesPerJob=$(( $nFilesPerJob+1 ))
142			fi
143
144	paus	1.1	echo " n files all : $nFiles"
145			echo " n files/proc : $nFilesPerJob"
146
147			i=1
148			next=1
149			last=$nFilesPerJob
150
151	paus	1.3	# make sure condor is properly setup for us
152			if ! [ -z $CONDOR_LOCATION ]
153			then
154			unset CONDOR_LOCATION
155			export CONDOR_CONFIG=/usr/local/condor/etc/condor_config
156			fi
157
158			# loop over the condor jobs and submit them
159	paus	1.4	while [ $i -le $nCopyProcs ] && [ $last -le $nFiles ]
160	paus	1.1	do
161			if [ $i == $nCopyProcs ]
162			then
163			last=$nFiles
164			fi
165
166			# say what we are going to submit
167			echo " downloadFiles.sh $dataDir $book $dataset $target $condorOutput $next $last"
168
169			logFile=`echo download:$book/$dataset/${next}-${last}.txt \| tr '/' '+'`
170			logFile=/tmp/$logFile
171			rm -f $logFile
172
173			# prepare the condor_submit files
174			cat > submit_$$.cmd <<EOF
175			Universe = vanilla
176	ceballos	1.6	Requirements = ( (Arch == "X86_64") && (OpSys == "LINUX" \|\| Arch == "INTEL") && (Disk >= DiskUsage) && ((Memory * 1024) >= ImageSize) && (HasFileTransfer) )
177	paus	1.1	Notify_user = paus@mit.edu
178			Notification = Error
179			Executable = $script
180			Arguments = $dataDir $book $dataset $target $condorOutput $next $last
181			Rank = Mips
182			GetEnv = True
183			Input = /dev/null
184			Output = $condorOutput/$book/$dataset/${next}-${last}.out
185			Error = $condorOutput/$book/$dataset/${next}-${last}.err
186			Log = $logFile
187			should_transfer_files = YES
188			when_to_transfer_output = ON_EXIT
189	paus	1.3
190			+AccountingGroup = "group_cmsuser.cmsu0284"
191
192	paus	1.1	Queue
193			EOF
194
195	paus	1.3	#+x509userproxysubject = $DN
196
197	paus	1.1	# submit the jobs
198			condor_submit submit_$$.cmd >& /dev/null #>& lastSub
199			#cat submit_$$.cmd
200			rm submit_$$.cmd
201
202			# update counters
203			next=$(( $next + $nFilesPerJob ))
204			last=$(( $last + $nFilesPerJob ))
205			i=$(( $i + 1 ))
206			done
207
208			exit 0