| 4 |
|
import os, popen2 |
| 5 |
|
from ProdCommon.Storage.SEAPI.SElement import SElement, FullPath |
| 6 |
|
from ProdCommon.Storage.SEAPI.SBinterface import * |
| 7 |
+ |
from ProdCommon.Storage.SEAPI.Exceptions import * |
| 8 |
|
|
| 9 |
|
|
| 10 |
|
class cmscp: |
| 28 |
|
|
| 29 |
|
#set default |
| 30 |
|
self.params = {"source":'', "destination":'', "inputFileList":'', "outputFileList":'', \ |
| 31 |
< |
"protocol":'', "option":'', "middleware":'', "srm_version":''} |
| 31 |
> |
"protocol":'', "option":'', "middleware":'', "srm_version":'srmv2'} |
| 32 |
|
self.debug = 0 |
| 33 |
|
|
| 34 |
|
self.params.update( args ) |
| 89 |
|
which depend on scheduler |
| 90 |
|
""" |
| 91 |
|
# default To be used with "middleware" |
| 92 |
< |
lcgOpt='-b -D srmv2 --vo cms -t 2400 --verbose' |
| 93 |
< |
srmOpt='-debug=true -report ./srmcp.report -retry_timeout 480000 -retry_num 3' |
| 92 |
> |
lcgOpt={'srmv1':'-b -D srmv1 --vo cms -t 2400 --verbose', |
| 93 |
> |
'srmv2':'-b -D srmv2 --vo cms -t 2400 --verbose'} |
| 94 |
> |
srmOpt={'srmv1':'-debug=true -report ./srmcp.report -retry_timeout 480000 -retry_num 3 -streams_num=1 ', |
| 95 |
> |
'srmv2':'-debug=true -report ./srmcp.report -retry_timeout 480000 -retry_num 3 -srm_protocol_version 2 '} |
| 96 |
|
rfioOpt='' |
| 97 |
|
|
| 98 |
|
if middleware.lower() in ['osg','lcg']: |
| 99 |
< |
supported_protocol = [('srm-lcg',lcgOpt),\ |
| 100 |
< |
('srmv2',srmOpt)] |
| 99 |
> |
supported_protocol = [('srm-lcg',lcgOpt[params['srm_version']]),\ |
| 100 |
> |
(params['srm_version'],srmOpt[params['srm_version']])] |
| 101 |
|
elif middleware.lower() in ['lsf','caf']: |
| 102 |
|
supported_protocol = [('rfio',rfioOpt)] |
| 103 |
|
else: |
| 184 |
|
try : |
| 185 |
|
sbi = SBinterface( Source_SE, Destination_SE ) |
| 186 |
|
sbi_dest = SBinterface(Destination_SE) |
| 187 |
< |
except Exception, ex: |
| 188 |
< |
msg = '' |
| 186 |
< |
if self.debug : msg = str(ex)+'\n' |
| 187 |
> |
except ProtocolMismatch, ex: |
| 188 |
> |
msg = str(ex)+'\n' |
| 189 |
|
msg += "ERROR : Unable to create SBinterface with %s protocol\n"%protocol |
| 190 |
|
raise msg |
| 191 |
|
|
| 207 |
|
""" |
| 208 |
|
try: |
| 209 |
|
interface = SElement( FullPath(endpoint), protocol ) |
| 210 |
< |
except Exception, ex: |
| 210 |
> |
except ProtocolUnknown, ex: |
| 211 |
|
msg = '' |
| 212 |
|
if self.debug : msg = str(ex)+'\n' |
| 213 |
|
msg += "ERROR : Unable to create interface with %s protocol\n"%protocol |
| 215 |
|
|
| 216 |
|
return interface |
| 217 |
|
|
| 216 |
– |
def checkDir(self, Destination_SE, protocol): |
| 217 |
– |
''' |
| 218 |
– |
ToBeImplemented NEEDED for castor |
| 219 |
– |
''' |
| 220 |
– |
return |
| 221 |
– |
|
| 218 |
|
def createDir(self, Destination_SE, protocol): |
| 219 |
|
""" |
| 220 |
|
Create remote dir for gsiftp REALLY TEMPORARY |
| 226 |
|
action = SBinterface( Destination_SE ) |
| 227 |
|
action.createDir() |
| 228 |
|
if self.debug: print "The directory has been created using protocol %s\n"%protocol |
| 229 |
< |
except Exception, ex: |
| 230 |
< |
msg = '' |
| 231 |
< |
if self.debug : msg = str(ex)+'\n' |
| 232 |
< |
msg = "ERROR: problem with the directory creation using %s protocol \n"%protocol |
| 229 |
> |
except TransferException, ex: |
| 230 |
> |
msg = str(ex) |
| 231 |
> |
if self.debug : |
| 232 |
> |
msg += str(ex.detail)+'\n' |
| 233 |
> |
msg += str(ex.out)+'\n' |
| 234 |
> |
msg_rep = "ERROR: problem with the directory creation using %s protocol \n"%protocol |
| 235 |
|
ErCode = '60316' |
| 236 |
+ |
res = self.updateReport('', ErCode, msg_rep ) |
| 237 |
+ |
self.finalReport( res ) |
| 238 |
+ |
raise msg |
| 239 |
+ |
except OperationException, ex: |
| 240 |
+ |
msg = str(ex) |
| 241 |
+ |
if self.debug : msg += str(ex.detail)+'\n' |
| 242 |
+ |
print msg |
| 243 |
|
|
| 244 |
|
return ErCode, msg |
| 245 |
|
|
| 251 |
|
msg = '' |
| 252 |
|
try: |
| 253 |
|
check = sbi.checkExists(filetocopy) |
| 254 |
< |
except Exception, ex: |
| 255 |
< |
msg = '' |
| 256 |
< |
if self.debug : msg = str(ex)+'\n' |
| 254 |
> |
except OperationException, ex: |
| 255 |
> |
msg = str(ex) |
| 256 |
> |
if self.debug : |
| 257 |
> |
msg += str(ex.detail)+'\n' |
| 258 |
> |
msg += str(ex.out)+'\n' |
| 259 |
|
msg +='problems checkig if file already exist' |
| 260 |
|
raise msg |
| 261 |
+ |
except WrongOption, ex: |
| 262 |
+ |
msg = str(ex) |
| 263 |
+ |
if self.debug : |
| 264 |
+ |
msg += str(ex.detail)+'\n' |
| 265 |
+ |
msg += str(ex.out)+'\n' |
| 266 |
+ |
raise msg |
| 267 |
|
if check : |
| 268 |
|
ErCode = '60303' |
| 269 |
|
msg = "file %s already exist"%filetocopy |
| 290 |
|
|
| 291 |
|
try: |
| 292 |
|
sbi.copy( source_file , dest_file , opt = option) |
| 293 |
< |
#if self.protocol == 'srm' : self.checkSize( sbi, filetocopy ) ## TODO |
| 294 |
< |
except Exception, ex: |
| 295 |
< |
msg = '' |
| 296 |
< |
if self.debug : msg = str(ex)+'\n' |
| 297 |
< |
msg = "Problem copying %s file" % filetocopy |
| 293 |
> |
except TransferException, ex: |
| 294 |
> |
msg = str(ex) |
| 295 |
> |
if self.debug : |
| 296 |
> |
msg += str(ex.detail)+'\n' |
| 297 |
> |
msg += str(ex.out)+'\n' |
| 298 |
> |
msg += "Problem copying %s file" % filetocopy |
| 299 |
|
ErCode = '60307' |
| 300 |
< |
|
| 300 |
> |
except WrongOption, ex: |
| 301 |
> |
msg = str(ex) |
| 302 |
> |
if self.debug : |
| 303 |
> |
msg += str(ex.detail)+'\n' |
| 304 |
> |
msg += str(ex.out)+'\n' |
| 305 |
> |
raise msg |
| 306 |
> |
|
| 307 |
> |
## TO BE IMPLEMENTED if NEEDED |
| 308 |
> |
## NOTE: SE API Already available |
| 309 |
> |
# if self.protocol.find('srm') : self.checkSize( sbi, filetocopy ) |
| 310 |
> |
|
| 311 |
|
return ErCode, msg |
| 312 |
|
|
| 289 |
– |
''' |
| 290 |
– |
def checkSize() |
| 291 |
– |
""" |
| 292 |
– |
Using srm needed a check of the ouptut file size. |
| 293 |
– |
""" |
| 294 |
– |
|
| 295 |
– |
echo "--> remoteSize = $remoteSize" |
| 296 |
– |
## for local file |
| 297 |
– |
localSize=$(stat -c%s "$path_out_file") |
| 298 |
– |
echo "--> localSize = $localSize" |
| 299 |
– |
if [ $localSize != $remoteSize ]; then |
| 300 |
– |
echo "Local fileSize $localSize does not match remote fileSize $remoteSize" |
| 301 |
– |
echo "Copy failed: removing remote file $destination" |
| 302 |
– |
srmrm $destination |
| 303 |
– |
cmscp_exit_status=60307 |
| 304 |
– |
|
| 305 |
– |
|
| 306 |
– |
echo "Problem copying $path_out_file to $destination with srmcp command" |
| 307 |
– |
StageOutExitStatusReason='remote and local file dimension not match' |
| 308 |
– |
echo "StageOutReport = `cat ./srmcp.report`" |
| 309 |
– |
''' |
| 313 |
|
def backup(self): |
| 314 |
|
""" |
| 315 |
|
Check infos from TFC using existing api obtaining: |
| 339 |
|
outFile = open('cmscpReport.sh',"a") |
| 340 |
|
cmscp_exit_status = 0 |
| 341 |
|
txt = '' |
| 342 |
< |
for file, dict in results.iteritems(): |
| 343 |
< |
if dict['lfn']=='': |
| 344 |
< |
lfn = '$LFNBaseName/'+os.path.basename(file) |
| 345 |
< |
se = '$SE' |
| 342 |
> |
for file, dict in results.iteritems(): |
| 343 |
> |
if file: |
| 344 |
> |
if dict['lfn']=='': |
| 345 |
> |
lfn = '$LFNBaseName/'+os.path.basename(file) |
| 346 |
> |
se = '$SE' |
| 347 |
> |
else: |
| 348 |
> |
lfn = dict['lfn']+os.pat.basename(file) |
| 349 |
> |
se = dict['se'] |
| 350 |
> |
#dict['lfn'] # to be implemented |
| 351 |
> |
txt += 'echo "Report for File: '+file+'"\n' |
| 352 |
> |
txt += 'echo "LFN: '+lfn+'"\n' |
| 353 |
> |
txt += 'echo "StorageElement: '+se+'"\n' |
| 354 |
> |
txt += 'echo "StageOutExitStatusReason ='+dict['reason']+'" | tee -a $RUNTIME_AREA/$repo\n' |
| 355 |
> |
txt += 'echo "StageOutSE = '+se+'" >> $RUNTIME_AREA/$repo\n' |
| 356 |
> |
if dict['erCode'] != '0': |
| 357 |
|
else: |
| 358 |
< |
lfn = dict['lfn']+os.pat.basename(file) |
| 359 |
< |
se = dict['se'] |
| 360 |
< |
#dict['lfn'] # to be implemented |
| 347 |
< |
txt += 'echo "Report for File: '+file+'"\n' |
| 348 |
< |
txt += 'echo "LFN: '+lfn+'"\n' |
| 349 |
< |
txt += 'echo "StorageElement: '+se+'"\n' |
| 350 |
< |
txt += 'echo "StageOutExitStatusReason ='+dict['reason']+'" | tee -a $RUNTIME_AREA/$repo\n' |
| 351 |
< |
txt += 'echo "StageOutSE = '+se+'" >> $RUNTIME_AREA/$repo\n' |
| 352 |
< |
if dict['erCode'] != '0': |
| 353 |
< |
cmscp_exit_status = dict['erCode'] |
| 358 |
> |
if dict['erCode'] != '0': |
| 359 |
> |
cmscp_exit_status = dict['erCode'] |
| 360 |
> |
cmscp_exit_status = dict['erCode'] |
| 361 |
|
txt += '\n' |
| 362 |
|
txt += 'export StageOutExitStatus='+str(cmscp_exit_status)+'\n' |
| 363 |
|
txt += 'echo "StageOutExitStatus = '+str(cmscp_exit_status)+'" | tee -a $RUNTIME_AREA/$repo\n' |
| 416 |
|
try: |
| 417 |
|
cmscp_ = cmscp(dictArgs) |
| 418 |
|
cmscp_.run() |
| 419 |
< |
except: |
| 420 |
< |
pass |
| 419 |
> |
except Exception, ex : |
| 420 |
> |
print str(ex) |
| 421 |
|
|
