ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/UserCode/MitProd/Processing/bin/downloadSample.sh
Revision: 1.11
Committed: Tue Feb 28 11:54:35 2012 UTC (13 years, 2 months ago) by paus
Content type: application/x-sh
Branch: MAIN
CVS Tags: Mit_028a, Mit_028, Mit_027a, Mit_027, Mit_026, Mit_025e, Mit_025d
Changes since 1.10: +43 -25 lines
Log Message:
Last updates.

File Contents

# Content
1 #!/bin/bash
2 #---------------------------------------------------------------------------------------------------
3 # Download a list of files
4 #---------------------------------------------------------------------------------------------------
5
6 # Read the arguments
7 echo ""
8 echo " downloadSample.sh $*"
9 echo ""
10 dataDir=$1; shift
11 book=$1; shift
12 dataset=$1; shift
13 target=$1; shift
14 nCopyProcs=$1; shift
15 condorOutput=$1; shift
16 onlyMissing=$1; shift
17
18 DN=`grid-proxy-info -subject`
19
20 # Prepare environment
21 echo " "
22 echo " Process dataset: $dataset of book: $book"
23 echo " in directory : $dataDir"
24 echo " to target : $target"
25 echo " n copy procs : $nCopyProcs"
26 echo " condor output: $condorOutput"
27 echo " only missing : $onlyMissing"
28
29 mkdir -p $condorOutput/$book/$dataset
30 makedir --exe $target
31 makedir --exe $target/$book
32 makedir --exe $target/$book/$dataset
33 script=`which downloadFiles.sh`
34
35 # cleanup our lists and remake a clean one
36 #echo "rm -f $condorOutput/$book/$dataset/fileList*.$$.txt*"
37 rm -f $condorOutput/$book/$dataset/fileList*.$$.txt*
38
39 # make list of all local files
40 if [ "`echo $HOSTNAME | grep mit.edu`" != "" ] && \
41 ( [ "`echo $dataDir | grep /castor/cern.ch`" != "" ] || \
42 [ "`echo $target | grep /castor/cern.ch`" != "" ] )
43 then
44 opt="--simple"
45 else
46 opt=""
47 fi
48
49 list $opt $dataDir/$book/$dataset | sort > $condorOutput/$book/$dataset/fileList-all.$$.txt-bak
50
51 # Make sure there are kerberos and globus tickets available
52 id=`id -u`
53 mkdir -p ~/.krb5/
54 cp /tmp/x509up_u${id} ~/.krb5/
55 KRB5CCNAME=`klist -5 | grep 'Ticket cache:' | cut -d' ' -f 3`
56 if ! [ -z $KRB5CCNAME ]
57 then
58 chmod 0 ~/.krb5
59 chmod u=rwx ~/.krb5
60 file=`echo $KRB5CCNAME | cut -d: -f2`
61 if [ -f "$file" ]
62 then
63 cp $file ~/.krb5/ticket
64 else
65 echo " ERROR -- missing kerberos ticket ($KRB5CCNAME)."
66 exit 1
67 fi
68 else
69 echo " ERROR -- missing kerberos ticket ($KRB5CCNAME)."
70 fi
71
72 # make list of all remote files
73 rm -f $condorOutput/$book/$dataset/fileList-all.$$.txt
74 touch $condorOutput/$book/$dataset/fileList-all.$$.txt
75
76 cat $condorOutput/$book/$dataset/fileList-all.$$.txt-bak | grep root | sort | \
77 while read line
78 do
79 size=`echo $line | tr -s ' ' | cut -d ' ' -f 1`
80 file=`echo $line | tr -s ' ' | cut -d ' ' -f 2`
81 file=`basename $file`
82 echo "$size $file" >> $condorOutput/$book/$dataset/fileList-all.$$.txt
83 done
84
85 # make list of all local files
86 if [ "`echo $HOSTNAME | grep mit.edu`" != "" ] && \
87 ( [ "`echo $dataDir | grep /castor/cern.ch`" != "" ] || \
88 [ "`echo $target | grep /castor/cern.ch`" != "" ] )
89 then
90 opt="--simple"
91 else
92 opt=""
93 fi
94
95 dcache=/pnfs/cmsaf.mit.edu/t2bat/cms/store/user/paus
96 list $opt $dcache/$book/$dataset $target/$book/$dataset | grep root | sort \
97 > $condorOutput/$book/$dataset/fileList-done.$$.txt
98
99 diff -y $condorOutput/$book/$dataset/fileList-all.$$.txt \
100 $condorOutput/$book/$dataset/fileList-done.$$.txt > diff.$$
101 echo ""
102 echo " Files different in size: "
103 grep \| diff.$$
104 echo ""
105 echo " Files available in all and not done: "
106 grep \< diff.$$
107 echo ""
108 echo " Files done but not listed in all available: "
109 grep \> diff.$$
110 echo ""
111 rm diff.$$
112
113 # make list of missing files
114 rm -f $condorOutput/$book/$dataset/fileList.$$.txt
115 touch $condorOutput/$book/$dataset/fileList.$$.txt
116
117 cat $condorOutput/$book/$dataset/fileList-all.$$.txt | grep root | \
118 while read line
119 do
120 size=`echo $line | tr -s ' ' | cut -d ' ' -f 1`
121 file=`echo $line | tr -s ' ' | cut -d ' ' -f 2`
122 exists=`grep "$file" $condorOutput/$book/$dataset/fileList-done.$$.txt`
123 if [ "$exists" == "" ]
124 then
125 echo " -missing-- $file with $size bytes"
126 echo "$size $file" >> $condorOutput/$book/$dataset/fileList.$$.txt
127 # else
128 # echo " -exists--- $file with $size bytes - exists"
129 else
130 # now check that size matches
131 test=`grep "$size $file" $condorOutput/$book/$dataset/fileList-done.$$.txt`
132 if [ "$test" == "" ]
133 then
134 if [ "$onlyMissing" == "" ]
135 then
136 echo " -fileSize- $exists (remote: $size)"
137 echo "$size $file" >> $condorOutput/$book/$dataset/fileList.$$.txt
138 fi
139 fi
140 fi
141 done
142
143 nAll=`wc -l $condorOutput/$book/$dataset/fileList-all.$$.txt | cut -d ' ' -f1`
144 nMissing=`wc -l $condorOutput/$book/$dataset/fileList.$$.txt | cut -d ' ' -f1`
145 nDone=`wc -l $condorOutput/$book/$dataset/fileList-done.$$.txt | cut -d ' ' -f1`
146 echo ""
147 echo " Download Summary "
148 echo " All $nAll"
149 echo " Done $nDone"
150 echo " Missing $nMissing"
151 echo ""
152
153 # construct our job
154 nFiles=`wc -l $condorOutput/$book/$dataset/fileList.$$.txt | cut -d ' ' -f1`
155 if [ "$nFiles" == "" ] || [ "$nFiles" == "0" ]
156 then
157 echo " "
158 echo " No more files to download. EXIT."
159 exit 0
160 elif [ $nFiles -lt $nCopyProcs ]
161 then
162 nCopyProcs=$nFiles
163 fi
164 # how many files per job?
165 nFilesPerJob=$(( $nFiles/$nCopyProcs ))
166 nFilesTmp=$(( $nFilesPerJob*$nCopyProcs ))
167 if [ $nFilesPerJob == 1 ] && [ $nFiles -gt $nCopyProcs ]
168 then
169 nFilesPerJob=2
170 elif [ $nFilesTmp -lt $nFiles ]
171 then
172 nFilesPerJob=$(( $nFilesPerJob+1 ))
173 fi
174
175 echo " n files to copy: $nFiles"
176 echo " n files/proc : $nFilesPerJob"
177
178 i=1
179 next=1
180 last=$nFilesPerJob
181
182 # make sure condor is properly setup for us
183 if ! [ -z $CONDOR_LOCATION ]
184 then
185 unset CONDOR_LOCATION
186 export CONDOR_CONFIG=/usr/local/condor/etc/condor_config
187 fi
188
189 # loop over the condor jobs and submit them
190 while [ $i -le $nCopyProcs ] && [ $last -le $nFiles ]
191 do
192 if [ $i == $nCopyProcs ]
193 then
194 last=$nFiles
195 fi
196
197 # say what we are going to submit
198 echo " downloadFiles.sh $dataDir $book $dataset $target $condorOutput $$ $next $last"
199
200 logFile=`echo download:$book/$dataset/${next}-${last}.txt | tr '/' '+'`
201 logFile=/tmp/$logFile
202 rm -f $logFile
203
204 # prepare the condor_submit files
205 cat > submit_$$.cmd <<EOF
206 Universe = vanilla
207 Requirements = ( (Arch == "INTEL") && (Disk >= DiskUsage) && ((Memory * 1024) >= ImageSize) && (HasFileTransfer) )
208 Notify_user = paus@mit.edu
209 Notification = Error
210 Executable = $script
211 Arguments = $dataDir $book $dataset $target $condorOutput $$ $next $last
212 Rank = Mips
213 GetEnv = True
214 Input = /dev/null
215 Output = $condorOutput/$book/$dataset/${next}-${last}.out
216 Error = $condorOutput/$book/$dataset/${next}-${last}.err
217 Log = $logFile
218 should_transfer_files = YES
219 when_to_transfer_output = ON_EXIT
220
221 +AccountingGroup = "group_cmsuser.cmsu0284"
222
223 Queue
224 EOF
225
226 #+x509userproxysubject = $DN
227
228 # submit the jobs
229 condor_submit submit_$$.cmd >& /dev/null #>& lastSub
230 #cat submit_$$.cmd
231 rm submit_$$.cmd
232
233 # update counters
234 next=$(( $next + $nFilesPerJob ))
235 last=$(( $last + $nFilesPerJob ))
236 i=$(( $i + 1 ))
237 done
238
239 exit 0