1 |
nsmirnov |
1.1 |
#!/bin/sh
|
2 |
spiga |
1.65 |
|
3 |
|
|
#CRAB title
|
4 |
|
|
|
5 |
ewv |
1.54 |
#
|
6 |
nsmirnov |
1.1 |
# HEAD
|
7 |
|
|
#
|
8 |
|
|
#
|
9 |
slacapra |
1.45 |
echo "Running $0 with $# positional parameters: $*"
|
10 |
corvo |
1.11 |
|
11 |
slacapra |
1.31 |
function cmscp {
|
12 |
|
|
## safe copy of local file in current directory to remote SE via srmcp, including success checking
|
13 |
|
|
## input:
|
14 |
|
|
## $1 local file (with respect to current working directory)
|
15 |
|
|
## $2 remote SE
|
16 |
|
|
## $3 remote SE_PATH (absolute)
|
17 |
|
|
## $4 remote file name
|
18 |
|
|
## $5 grid environment: LCG (default) | OSG
|
19 |
|
|
## output:
|
20 |
|
|
## return 0 if all ok
|
21 |
fanzago |
1.50 |
## return 60307 if srmcp failed
|
22 |
|
|
## return 60303 if file already exists in the SE
|
23 |
slacapra |
1.31 |
###########################
|
24 |
|
|
if [ $# -le 4 ]; then
|
25 |
|
|
echo -e "\t$0 usage:"
|
26 |
spiga |
1.52 |
echo -e "\t$0 source <remote SE> <remote SE PATH> <remote output file name> <srm version 1(default)|2> <grid env: LCG(default)|OSG>"
|
27 |
slacapra |
1.31 |
exit 1
|
28 |
ewv |
1.43 |
fi
|
29 |
fanzago |
1.44 |
path_out_file=$1
|
30 |
|
|
echo "path_out_file = $path_out_file"
|
31 |
slacapra |
1.31 |
SE=$2
|
32 |
fanzago |
1.35 |
SE_PATH=$3
|
33 |
fanzago |
1.44 |
name_out_file=$4
|
34 |
spiga |
1.52 |
srm_ver=$5
|
35 |
slacapra |
1.31 |
middleware='LCG'
|
36 |
spiga |
1.52 |
if [ $# == 6 ]; then
|
37 |
|
|
middleware=$6
|
38 |
slacapra |
1.31 |
fi
|
39 |
|
|
|
40 |
|
|
# Set OSG certificates directory
|
41 |
|
|
if [ $middleware == OSG ]; then
|
42 |
gutsche |
1.32 |
echo "source OSG GRID setup script"
|
43 |
|
|
source $OSG_GRID/setup.sh
|
44 |
slacapra |
1.31 |
fi
|
45 |
|
|
|
46 |
|
|
## do the actual copy
|
47 |
fanzago |
1.44 |
destination=srm://${SE}:8443${SE_PATH}$name_out_file
|
48 |
fanzago |
1.35 |
echo "destination = $destination"
|
49 |
|
|
|
50 |
spiga |
1.62 |
################ lcg-utils ##########################
|
51 |
|
|
if [ $srm_ver -eq 1 ] ; then
|
52 |
ewv |
1.63 |
lcgOpt=" -b -D srmv1 --vo $VO -t 2400 --verbose "
|
53 |
spiga |
1.62 |
else
|
54 |
afanfani |
1.68 |
Dsrm_ver="2"
|
55 |
ewv |
1.63 |
lcgOpt=" -b -D srmv2 --vo $VO -t 2400 --verbose "
|
56 |
spiga |
1.62 |
fi
|
57 |
spiga |
1.66 |
echo "lcg-cp --version "
|
58 |
|
|
lcg-cp --version
|
59 |
|
|
echo "---------------- "
|
60 |
spiga |
1.62 |
cmd="lcg-cp $lcgOpt file://$path_out_file $destination"
|
61 |
|
|
echo $cmd
|
62 |
|
|
exitstring=`$cmd 2>&1`
|
63 |
|
|
cmscp_exit_status=$?
|
64 |
|
|
if [ $cmscp_exit_status -ne 0 ]; then
|
65 |
|
|
cmscp_exit_status=60307
|
66 |
|
|
echo "Problem copying $path_out_file to $destination with lcg-cp command"
|
67 |
afanfani |
1.68 |
echo "Error message: $exitstring "
|
68 |
spiga |
1.62 |
StageOutExitStatusReason=$exitstring
|
69 |
afanfani |
1.68 |
cmd="lcg-ls -D srmv${Dsrm_ver} $destination"
|
70 |
spiga |
1.62 |
tmpstring=`$cmd 2>&1`
|
71 |
ewv |
1.73 |
echo $tmpstring | grep 'not found'
|
72 |
spiga |
1.62 |
exit_status=$?
|
73 |
|
|
if [ $exit_status -eq 0 ]; then
|
74 |
fanzago |
1.56 |
cmscp_exit_status=60303
|
75 |
spiga |
1.52 |
StageOutExitStatusReason='file already exists'
|
76 |
spiga |
1.62 |
fi
|
77 |
|
|
else
|
78 |
|
|
StageOutExitStatusReason='copy ok with lcg utils'
|
79 |
|
|
fi
|
80 |
|
|
|
81 |
|
|
############# now try srm utils ##############
|
82 |
ewv |
1.63 |
if [ $cmscp_exit_status -ne 0 ]; then
|
83 |
spiga |
1.62 |
opt=" -debug=true -report ./srmcp.report "
|
84 |
|
|
opt="${opt} -retry_timeout 480000 -retry_num 3 "
|
85 |
|
|
|
86 |
|
|
################ srmv2 ##########################
|
87 |
|
|
if [ $srm_ver -eq 2 ] || [ $srm_ver -eq 0 ]; then
|
88 |
|
|
# if [ $srm_ver -eq 2 ] && [ $srm_ver -ne 1 ] ; then
|
89 |
afanfani |
1.68 |
## unset SRM_PATH
|
90 |
|
|
unset SRM_PATH
|
91 |
ewv |
1.63 |
echo "--> Check if the file already exists in the storage element $SE, using SRM2"
|
92 |
spiga |
1.62 |
srmls -retry_num 0 $destination | grep 'does not exist' >/dev/null
|
93 |
|
|
if [ $? -eq 0 ]; then
|
94 |
|
|
echo "Starting to copy the output to $SE using srmv2"
|
95 |
ewv |
1.63 |
cmd="srmcp -srm_protocol_version 2 $opt file:///$path_out_file $destination"
|
96 |
spiga |
1.62 |
echo $cmd
|
97 |
|
|
exitstring=`$cmd 2>&1`
|
98 |
|
|
cmscp_exit_status=$?
|
99 |
|
|
if [ $cmscp_exit_status -eq 0 ]; then
|
100 |
ewv |
1.64 |
remoteMetadata=(`srmls -retry_num=0 $destination | grep -v WARNING 2>/dev/null`)
|
101 |
spiga |
1.62 |
remoteSize=`echo ${remoteMetadata}`
|
102 |
|
|
echo "--> remoteSize = $remoteSize"
|
103 |
|
|
## for local file
|
104 |
|
|
localSize=$(stat -c%s "$path_out_file")
|
105 |
|
|
echo "--> localSize = $localSize"
|
106 |
|
|
if [ $localSize != $remoteSize ]; then
|
107 |
|
|
echo "Local fileSize $localSize does not match remote fileSize $remoteSize"
|
108 |
|
|
echo "Copy failed: removing remote file $destination"
|
109 |
|
|
srmrm $destination
|
110 |
|
|
cmscp_exit_status=60307
|
111 |
|
|
echo "Problem copying $path_out_file to $destination with srmcp command"
|
112 |
|
|
StageOutExitStatusReason='remote and local file dimension not match'
|
113 |
|
|
echo "StageOutReport = `cat ./srmcp.report`"
|
114 |
|
|
fi
|
115 |
|
|
StageOutExitStatusReason='copy ok with srm utils'
|
116 |
|
|
else
|
117 |
fanzago |
1.56 |
cmscp_exit_status=60307
|
118 |
spiga |
1.52 |
echo "Problem copying $path_out_file to $destination with srmcp command"
|
119 |
spiga |
1.62 |
StageOutExitStatusReason=$exitstring
|
120 |
spiga |
1.52 |
echo "StageOutReport = `cat ./srmcp.report`"
|
121 |
|
|
fi
|
122 |
|
|
else
|
123 |
spiga |
1.62 |
cmscp_exit_status=60303
|
124 |
|
|
StageOutExitStatusReason='file already exists'
|
125 |
spiga |
1.52 |
fi
|
126 |
|
|
fi
|
127 |
|
|
|
128 |
spiga |
1.62 |
if [ $srm_ver -eq 1 ] ; then
|
129 |
ewv |
1.63 |
echo "--> Check if the file already exists in the storage element $SE, using SRM1"
|
130 |
spiga |
1.62 |
srm-get-metadata -retry_num 0 $destination
|
131 |
|
|
if [ $? -eq 0 ]; then
|
132 |
|
|
cmscp_exit_status=60303
|
133 |
|
|
StageOutExitStatusReason='file already exists'
|
134 |
|
|
else
|
135 |
|
|
echo "Starting copy of the output to $SE, middleware is $middleware"
|
136 |
|
|
cmd="srmcp $opt -streams_num=1 file:///$path_out_file $destination"
|
137 |
|
|
echo $cmd
|
138 |
|
|
exitstring=`$cmd 2>&1`
|
139 |
|
|
cmscp_exit_status=$?
|
140 |
|
|
if [ $cmscp_exit_status -eq 0 ]; then
|
141 |
|
|
## Put into an array the remote file metadata
|
142 |
|
|
remoteMetadata=(`srm-get-metadata -retry_num 0 $destination | grep -v WARNING`)
|
143 |
|
|
remoteSize=`echo ${remoteMetadata[5]}| tr -d :`
|
144 |
|
|
echo "--> remoteSize = $remoteSize"
|
145 |
|
|
## for local file
|
146 |
|
|
localSize=$(stat -c%s "$path_out_file")
|
147 |
|
|
echo "--> localSize = $localSize"
|
148 |
|
|
if [ $localSize != $remoteSize ]; then
|
149 |
|
|
echo "Local fileSize $localSize does not match remote fileSize $remoteSize"
|
150 |
|
|
echo "Copy failed: removing remote file $destination"
|
151 |
|
|
srm-advisory-delete $destination
|
152 |
fanzago |
1.56 |
cmscp_exit_status=60307
|
153 |
spiga |
1.52 |
echo "Problem copying $path_out_file to $destination with srmcp command"
|
154 |
|
|
StageOutExitStatusReason='remote and local file dimension not match'
|
155 |
|
|
echo "StageOutReport = `cat ./srmcp.report`"
|
156 |
spiga |
1.62 |
fi
|
157 |
|
|
StageOutExitStatusReason='copy ok with srm utils'
|
158 |
|
|
else
|
159 |
|
|
cmscp_exit_status=60307
|
160 |
|
|
echo "Problem copying $path_out_file to $destination with srmcp command"
|
161 |
|
|
StageOutExitStatusReason=$exitstring
|
162 |
|
|
echo "StageOutReport = `cat ./srmcp.report`"
|
163 |
spiga |
1.52 |
fi
|
164 |
fanzago |
1.35 |
fi
|
165 |
ewv |
1.43 |
fi
|
166 |
slacapra |
1.31 |
|
167 |
fanzago |
1.35 |
fi
|
168 |
ewv |
1.43 |
|
169 |
fanzago |
1.56 |
echo "StageOutExitStatus = $cmscp_exit_status" | tee -a $RUNTIME_AREA/$repo
|
170 |
|
|
echo "StageOutExitStatusReason = $StageOutExitStatusReason" | tee -a $RUNTIME_AREA/$repo
|
171 |
|
|
echo "StageOutSE = $SE" >> $RUNTIME_AREA/$repo\n
|
172 |
|
|
return $cmscp_exit_status
|
173 |
slacapra |
1.31 |
}
|
174 |
|
|
|
175 |
corvo |
1.11 |
dumpStatus() {
|
176 |
fanzago |
1.56 |
echo ">>> info for dashboard:"
|
177 |
|
|
echo "***** Cat $1 *****"
|
178 |
|
|
cat $1
|
179 |
|
|
echo "***** End Cat jobreport *****"
|
180 |
|
|
chmod a+x $RUNTIME_AREA/report.py
|
181 |
|
|
$RUNTIME_AREA/report.py $(cat $1)
|
182 |
|
|
rm -f $1
|
183 |
|
|
echo "MonitorJobID=`echo $MonitorJobID`" > $1
|
184 |
|
|
echo "MonitorID=`echo $MonitorID`" >> $1
|
185 |
|
|
}
|
186 |
|
|
|
187 |
fanzago |
1.57 |
|
188 |
|
|
### CRAB UPDATE THE FJR WITH WRAPPER_EXIT_CODE ###
|
189 |
|
|
update_fjr() {
|
190 |
|
|
if [ ! -s $RUNTIME_AREA/JobReportErrorCode.py ]; then
|
191 |
|
|
echo "WARNING: it is not possible to create crab_fjr.xml to final report"
|
192 |
|
|
else
|
193 |
|
|
echo "PYTHONPATH = $PYTHONPATH"
|
194 |
|
|
chmod a+x $RUNTIME_AREA/JobReportErrorCode.py
|
195 |
fanzago |
1.60 |
python $RUNTIME_AREA/JobReportErrorCode.py $RUNTIME_AREA/crab_fjr_$NJob.xml $job_exit_code $executable_exit_status
|
196 |
fanzago |
1.57 |
fi
|
197 |
|
|
}
|
198 |
ewv |
1.63 |
|
199 |
fanzago |
1.57 |
### REMOVE THE WORKING_DIR IN OSG SITES ###
|
200 |
|
|
remove_working_dir() {
|
201 |
|
|
cd $RUNTIME_AREA
|
202 |
fanzago |
1.69 |
echo ">>> working dir = $WORKING_DIR"
|
203 |
fanzago |
1.57 |
echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"
|
204 |
|
|
echo ">>> Remove working directory: $WORKING_DIR"
|
205 |
|
|
/bin/rm -rf $WORKING_DIR
|
206 |
|
|
if [ -d $WORKING_DIR ] ;then
|
207 |
|
|
echo "ERROR ==> OSG $WORKING_DIR could not be deleted on WN `hostname`"
|
208 |
|
|
job_exit_code=10017
|
209 |
|
|
fi
|
210 |
|
|
}
|
211 |
ewv |
1.63 |
|
212 |
spiga |
1.65 |
#CRAB func_exit
|
213 |
ewv |
1.73 |
|
214 |
|
|
if [ -d $_CONDOR_SCRATCH_DIR ]; then
|
215 |
|
|
ORIG_WD=`pwd`
|
216 |
|
|
cp ../default.tgz $_CONDOR_SCRATCH_DIR
|
217 |
|
|
cd $_CONDOR_SCRATCH_DIR
|
218 |
|
|
fi
|
219 |
nsmirnov |
1.1 |
|
220 |
fanzago |
1.56 |
RUNTIME_AREA=`pwd`
|
221 |
|
|
|
222 |
nsmirnov |
1.1 |
echo "Today is `date`"
|
223 |
ewv |
1.43 |
echo "Job submitted on host `hostname`"
|
224 |
nsmirnov |
1.1 |
uname -a
|
225 |
fanzago |
1.38 |
echo ">>> current directory (RUNTIME_AREA): `pwd`"
|
226 |
|
|
echo ">>> current directory content:"
|
227 |
nsmirnov |
1.1 |
ls -Al
|
228 |
fanzago |
1.38 |
echo ">>> current user: `id`"
|
229 |
|
|
echo ">>> voms proxy information:"
|
230 |
slacapra |
1.28 |
voms-proxy-info -all
|
231 |
|
|
|
232 |
corvo |
1.11 |
repo=jobreport.txt
|
233 |
fanzago |
1.9 |
|
234 |
corvo |
1.27 |
|
235 |
ewv |
1.63 |
#CRAB untar_software
|
236 |
fanzago |
1.57 |
|
237 |
nsmirnov |
1.1 |
#
|
238 |
fanzago |
1.38 |
# SETUP ENVIRONMENT
|
239 |
nsmirnov |
1.1 |
#
|
240 |
|
|
|
241 |
fanzago |
1.38 |
#CRAB setup_scheduler_environment
|
242 |
|
|
|
243 |
|
|
#CRAB setup_jobtype_environment
|
244 |
|
|
|
245 |
nsmirnov |
1.1 |
#
|
246 |
fanzago |
1.38 |
# END OF SETUP ENVIRONMENT
|
247 |
nsmirnov |
1.1 |
#
|
248 |
corvo |
1.25 |
#
|
249 |
|
|
# PREPARE AND RUN EXECUTABLE
|
250 |
|
|
#
|
251 |
|
|
|
252 |
fanzago |
1.38 |
#CRAB build_executable
|
253 |
fanzago |
1.6 |
|
254 |
corvo |
1.27 |
|
255 |
nsmirnov |
1.1 |
#
|
256 |
fanzago |
1.38 |
# END OF PREPARE AND RUN EXECUTABLE
|
257 |
nsmirnov |
1.1 |
#
|
258 |
|
|
|
259 |
|
|
#
|
260 |
fanzago |
1.12 |
# COPY INPUT
|
261 |
|
|
#
|
262 |
|
|
|
263 |
ewv |
1.43 |
#CRAB copy_input
|
264 |
fanzago |
1.12 |
|
265 |
ewv |
1.55 |
#
|
266 |
|
|
# Rewrite cfg or cfgpy file
|
267 |
|
|
#
|
268 |
|
|
|
269 |
|
|
#CRAB rewrite_cmssw_cfg
|
270 |
|
|
|
271 |
fanzago |
1.38 |
echo ">>> Executable $executable"
|
272 |
nsmirnov |
1.2 |
which $executable
|
273 |
nsmirnov |
1.1 |
res=$?
|
274 |
ewv |
1.43 |
if [ $res -ne 0 ];then
|
275 |
fanzago |
1.56 |
# echo "SET_EXE 1 ==> ERROR executable not found on WN `hostname`"
|
276 |
|
|
# echo "JOB_EXIT_STATUS = 50110"
|
277 |
|
|
# echo "JobExitStatus=50110" | tee -a $RUNTIME_AREA/$repo
|
278 |
|
|
# dumpStatus $RUNTIME_AREA/$repo
|
279 |
|
|
# exit
|
280 |
|
|
|
281 |
|
|
echo "ERROR ==> executable not found on WN `hostname`"
|
282 |
|
|
job_exit_code=50110
|
283 |
ewv |
1.63 |
func_exit
|
284 |
fanzago |
1.56 |
else
|
285 |
|
|
echo "ok executable found"
|
286 |
nsmirnov |
1.1 |
fi
|
287 |
|
|
|
288 |
fanzago |
1.56 |
#echo "SET_EXE 0 ==> ok executable found"
|
289 |
spiga |
1.4 |
|
290 |
fanzago |
1.56 |
echo "ExeStart=$executable" >> $RUNTIME_AREA/$repo
|
291 |
fanzago |
1.12 |
dumpStatus $RUNTIME_AREA/$repo
|
292 |
ewv |
1.54 |
#cat pset.py
|
293 |
fanzago |
1.38 |
echo ">>> $executable started at `date`"
|
294 |
spiga |
1.4 |
start_exe_time=`date +%s`
|
295 |
nsmirnov |
1.1 |
#CRAB run_executable
|
296 |
|
|
executable_exit_status=$?
|
297 |
gutsche |
1.23 |
stop_exe_time=`date +%s`
|
298 |
fanzago |
1.56 |
echo ">>> $executable ended at `date`"
|
299 |
|
|
|
300 |
|
|
#### dashboard add timestamp!
|
301 |
|
|
echo "ExeEnd=$executable" >> $RUNTIME_AREA/$repo
|
302 |
|
|
dumpStatus $RUNTIME_AREA/$repo
|
303 |
|
|
|
304 |
|
|
let "TIME_EXE = stop_exe_time - start_exe_time"
|
305 |
|
|
echo "TIME_EXE = $TIME_EXE sec"
|
306 |
|
|
echo "ExeTime=$TIME_EXE" >> $RUNTIME_AREA/$repo
|
307 |
gutsche |
1.23 |
|
308 |
ewv |
1.73 |
|
309 |
spiga |
1.72 |
#CRAB parse_report
|
310 |
nsmirnov |
1.1 |
|
311 |
|
|
|
312 |
|
|
#
|
313 |
nsmirnov |
1.3 |
# PROCESS THE PRODUCED RESULTS
|
314 |
nsmirnov |
1.1 |
#
|
315 |
|
|
|
316 |
nsmirnov |
1.3 |
#CRAB rename_output
|
317 |
nsmirnov |
1.1 |
|
318 |
ewv |
1.43 |
#CRAB copy_output
|
319 |
fanzago |
1.5 |
|
320 |
fanzago |
1.38 |
echo ">>> current dir: `pwd`"
|
321 |
|
|
echo ">>> current dir content:"
|
322 |
|
|
ls -Al
|
323 |
|
|
|
324 |
|
|
#CRAB modify_report
|
325 |
|
|
|
326 |
nsmirnov |
1.1 |
#
|
327 |
nsmirnov |
1.3 |
# END OF PROCESS THE PRODUCED RESULTS
|
328 |
nsmirnov |
1.1 |
#
|
329 |
|
|
|
330 |
ewv |
1.43 |
#CRAB clean_env
|
331 |
fanzago |
1.30 |
|
332 |
fanzago |
1.56 |
func_exit
|