1 |
< |
#!/usr/bin/env python |
1 |
> |
##!/usr/bin/env python |
2 |
|
|
3 |
|
import sys, os |
4 |
|
from ProdCommon.Storage.SEAPI.SElement import SElement, FullPath |
26 |
|
""" |
27 |
|
|
28 |
|
#set default |
29 |
< |
self.params = {"source":'', "destination":'', "inputFileList":'', "outputFileList":'', \ |
29 |
> |
self.params = {"source":'', "destination":'','destinationDir':'', "inputFileList":'', "outputFileList":'', \ |
30 |
|
"protocol":'', "option":'', "middleware":'', "srm_version":'srmv2'} |
31 |
|
self.debug = 0 |
32 |
|
|
38 |
|
""" |
39 |
|
check command line parameter |
40 |
|
""" |
41 |
– |
|
41 |
|
if 'help' in self.params.keys(): HelpOptions() |
42 |
|
if 'debug' in self.params.keys(): self.debug = 1 |
43 |
|
|
44 |
|
# source and dest cannot be undefined at same time |
45 |
|
if not self.params['source'] and not self.params['destination'] : |
46 |
|
HelpOptions() |
48 |
– |
|
47 |
|
# if middleware is not defined --> protocol cannot be empty |
48 |
|
if not self.params['middleware'] and not self.params['protocol'] : |
49 |
|
HelpOptions() |
50 |
|
|
51 |
|
# input file must be defined |
52 |
< |
if not self.params['inputFileList'] : HelpOptions() |
52 |
> |
if not self.params['inputFileList'] : |
53 |
> |
HelpOptions() |
54 |
|
else: |
55 |
|
file_to_copy=[] |
56 |
|
if self.params['inputFileList'].find(','): |
63 |
|
#### add check for outFiles |
64 |
|
#### add map {'inFileNAME':'outFileNAME'} to change out name |
65 |
|
|
66 |
< |
return |
66 |
> |
return msg |
67 |
|
|
68 |
|
def run( self ): |
69 |
|
""" |
70 |
|
Check if running on UI (no $middleware) or |
71 |
|
on WN (on the Grid), and take different action |
72 |
|
""" |
73 |
< |
|
74 |
< |
self.processOptions() |
73 |
> |
OptRes = self.processOptions() |
74 |
> |
if self.debug: print 'calling run() : \n' |
75 |
|
# stage out from WN |
76 |
|
if self.params['middleware'] : |
77 |
< |
results = self.stager(self.params['middleware'],self.params['inputFileList']) |
78 |
< |
self.finalReport(results) |
77 |
> |
results = self.stager(self.params['middleware'],self.params['inputFileList']) |
78 |
> |
self.finalReport(results) |
79 |
|
# Local interaction with SE |
80 |
|
else: |
81 |
< |
results = self.copy(self.params['inputFilesList'], self.params['protocol'], self.params['option'] ) |
82 |
< |
return results |
81 |
> |
results = self.copy(self.params['inputFileList'], self.params['protocol'], self.params['option'] ) |
82 |
> |
return results |
83 |
|
|
84 |
|
def setProtocol( self, middleware ): |
85 |
|
""" |
90 |
|
lcgOpt={'srmv1':'-b -D srmv1 -t 2400 --verbose', |
91 |
|
'srmv2':'-b -D srmv2 -t 2400 --verbose'} |
92 |
|
srmOpt={'srmv1':' -report ./srmcp.report -retry_timeout 480000 -retry_num 3 -streams_num=1 ', |
93 |
< |
'srmv2':' -report ./srmcp.report -retry_timeout 480000 -retry_num 3 '} |
93 |
> |
'srmv2':' -report=./srmcp.report -retry_timeout=480000 -retry_num=3 '} |
94 |
|
rfioOpt='' |
95 |
|
|
96 |
|
supported_protocol = None |
97 |
|
if middleware.lower() in ['osg','lcg','condor']: |
98 |
|
supported_protocol = [('srm-lcg',lcgOpt[self.params['srm_version']]),\ |
99 |
< |
(self.params['srm_version'],srmOpt[self.params['srm_version']])] |
99 |
> |
(self.params['srm_version'],srmOpt[self.params['srm_version']])] |
100 |
|
elif middleware.lower() in ['lsf','caf']: |
101 |
|
supported_protocol = [('rfio',rfioOpt)] |
102 |
|
else: |
105 |
|
pass |
106 |
|
return supported_protocol |
107 |
|
|
108 |
+ |
# def checkCopy(self, copy_results, list_files): |
109 |
+ |
""" |
110 |
+ |
#results={} |
111 |
+ |
list_retry = [] |
112 |
+ |
list_existing = [] |
113 |
+ |
list_ok = [] |
114 |
+ |
if copy_results.keys() == '': |
115 |
+ |
self.results.update(copy_results) |
116 |
+ |
else: |
117 |
+ |
for file, dict in copy_results.iteritems(): |
118 |
+ |
er_code = dict['erCode'] |
119 |
+ |
if er_code == '0': |
120 |
+ |
list_ok.append(file) |
121 |
+ |
reason = 'Copy succedeed with %s utils'%prot |
122 |
+ |
upDict = self.updateReport(file, er_code, reason) |
123 |
+ |
copy_results.update(upDict) |
124 |
+ |
elif er_code == '60303': list_existing.append( file ) |
125 |
+ |
else: list_retry.append( file ) |
126 |
+ |
results.update(copy_results) |
127 |
+ |
if len(list_ok) != 0: |
128 |
+ |
msg = 'Copy of %s succedeed with %s utils\n'%(str(list_ok),prot) |
129 |
+ |
if self.debug : print msg |
130 |
+ |
if len(list_ok) == len(list_files) : |
131 |
+ |
msg = 'Copy of all files succedeed\n' |
132 |
+ |
#break |
133 |
+ |
else: |
134 |
+ |
if self.debug : print 'Copy of files %s failed using %s...\n'%(str(list_retry)+str(list_existing),prot) |
135 |
+ |
#if len(list_retry): list_files = list_retry |
136 |
+ |
return list_retry, results |
137 |
+ |
|
138 |
+ |
""" |
139 |
|
def stager( self, middleware, list_files ): |
140 |
|
""" |
141 |
|
Implement the logic for remote stage out |
142 |
|
""" |
143 |
+ |
|
144 |
+ |
if self.debug: print 'stager() :\n' |
145 |
|
results={} |
146 |
|
for prot, opt in self.setProtocol( middleware ): |
147 |
< |
if self.debug: print 'Trying stage out with %s utils \n'%prot |
147 |
> |
if self.debug: print '\tTrying the stage out with %s utils \n'%prot |
148 |
|
copy_results = self.copy( list_files, prot, opt ) |
149 |
+ |
######## to define a new function checkCopy ################ |
150 |
+ |
#list_retry, self.results = self.checkCopy(copy_results, list_files) |
151 |
+ |
#def checkCopy (self, copy_results): |
152 |
+ |
# """ |
153 |
+ |
# """ |
154 |
+ |
# results={} |
155 |
|
list_retry = [] |
156 |
|
list_existing = [] |
157 |
|
list_ok = [] |
158 |
< |
if copy_results.keys() == '': |
158 |
> |
if copy_results.keys() == [''] or copy_results.keys() == '' : |
159 |
|
results.update(copy_results) |
160 |
|
else: |
161 |
|
for file, dict in copy_results.iteritems(): |
165 |
|
reason = 'Copy succedeed with %s utils'%prot |
166 |
|
upDict = self.updateReport(file, er_code, reason) |
167 |
|
copy_results.update(upDict) |
168 |
< |
elif er_code == '60303': list_existing.append( file ) |
168 |
> |
elif er_code in ['60303','60302']: list_existing.append( file ) |
169 |
|
else: list_retry.append( file ) |
170 |
|
results.update(copy_results) |
171 |
+ |
msg = '' |
172 |
|
if len(list_ok) != 0: |
173 |
< |
msg = 'Copy of %s succedeed with %s utils\n'%(str(list_ok),prot) |
135 |
< |
if self.debug : print msg |
173 |
> |
msg += '\tCopy of %s succedeed with %s utils\n'%(str(list_ok),prot) |
174 |
|
if len(list_ok) == len(list_files) : |
175 |
|
break |
176 |
|
else: |
177 |
< |
if self.debug : print 'Copy of files %s failed using %s...\n'%(str(list_retry)+str(list_existing),prot) |
177 |
> |
if self.debug: msg += '\tCopy of %s failed using %s...\n'%(str(list_retry)+str(list_existing),prot) |
178 |
|
if len(list_retry): list_files = list_retry |
179 |
|
else: break |
180 |
< |
|
180 |
> |
if self.debug : print msg |
181 |
> |
""" |
182 |
> |
if len(list_retry): |
183 |
> |
list_files = list_retry |
184 |
> |
#def backupCopy(list_retry) |
185 |
> |
print "in backup" |
186 |
> |
self.params['inputFilesList']=list_files |
187 |
> |
### copy backup |
188 |
> |
from ProdCommon.FwkJobRep.SiteLocalConfig import loadSiteLocalConfig |
189 |
> |
siteCfg = loadSiteLocalConfig() |
190 |
> |
#print siteCfg |
191 |
> |
seName = siteCfg.localStageOut.get("se-name", None) |
192 |
> |
#print "seName = ", seName |
193 |
> |
self.params['destination']=seName |
194 |
> |
#catalog = siteCfg.localStageOut.get("catalog", None) |
195 |
> |
#print "catalog = ", catalog |
196 |
> |
implName = siteCfg.localStageOut.get("command", None) |
197 |
> |
print "implName = ", implName |
198 |
> |
if (implName == 'srm'): |
199 |
> |
implName='srmv2' |
200 |
> |
self.params['protocol']=implName |
201 |
> |
tfc = siteCfg.trivialFileCatalog() |
202 |
> |
#print "tfc = ", tfc |
203 |
> |
print " self.params['inputFilesList'] = ", self.params['inputFilesList'] |
204 |
> |
file_backup=[] |
205 |
> |
for input in self.params['inputFilesList']: |
206 |
> |
### to add the correct lfn, passed as argument of cmscp function (--lfn xxxx) |
207 |
> |
file = '/store/'+input |
208 |
> |
pfn = tfc.matchLFN(tfc.preferredProtocol, file) |
209 |
> |
print "pfn = ", pfn |
210 |
> |
file_backup.append(pfn) |
211 |
> |
self.params['inputFilesList'] = file_backup |
212 |
> |
print "#########################################" |
213 |
> |
print "self.params['inputFilesList'] = ", self.params['inputFilesList'] |
214 |
> |
print "self.params['protocol'] = ", self.params['protocol'] |
215 |
> |
print "self.params['option'] = ", self.params['option'] |
216 |
> |
self.copy(self.params['inputFilesList'], self.params['protocol'], self.params['option']) |
217 |
> |
print "#########################################" |
218 |
> |
###list_retry, self.results = checkCopy(copy_results) |
219 |
> |
#check is something fails and created related dict |
220 |
> |
# backup = self.analyzeResults(results) |
221 |
> |
# if backup : |
222 |
> |
# msg = 'WARNING: backup logic is under implementation\n' |
223 |
> |
# #backupDict = self.backup() |
224 |
> |
# ### NOTE: IT MUST RETURN a DICT contains also LFN and SE Name |
225 |
> |
# results.update(backupDict) |
226 |
> |
# print msg |
227 |
> |
""" |
228 |
|
#### TODO Daniele |
229 |
|
#check is something fails and created related dict |
230 |
|
# backup = self.analyzeResults(results) |
241 |
|
""" |
242 |
|
Instantiate storage interface |
243 |
|
""" |
244 |
+ |
if self.debug : print 'initializeApi() :\n' |
245 |
|
self.source_prot = protocol |
246 |
|
self.dest_prot = protocol |
247 |
|
if not self.params['source'] : self.source_prot = 'local' |
250 |
|
Destination_SE = self.storageInterface( self.params['destination'], self.dest_prot ) |
251 |
|
|
252 |
|
if self.debug : |
253 |
< |
print '(source=%s, protocol=%s)'%(self.params['source'], self.source_prot) |
254 |
< |
print '(destination=%s, protocol=%s)'%(self.params['destination'], self.dest_prot) |
253 |
> |
msg = '\t(source=%s, protocol=%s)'%(self.params['source'], self.source_prot) |
254 |
> |
msg += '\t(destination=%s, protocol=%s)'%(self.params['destination'], self.dest_prot) |
255 |
> |
print msg |
256 |
|
|
257 |
|
return Source_SE, Destination_SE |
258 |
|
|
261 |
|
Make the real file copy using SE API |
262 |
|
""" |
263 |
|
if self.debug : |
264 |
< |
print 'copy(): using %s protocol'%protocol |
264 |
> |
msg = 'copy() :\n' |
265 |
> |
msg += '\tusing %s protocol\n'%protocol |
266 |
> |
print msg |
267 |
|
try: |
268 |
|
Source_SE, Destination_SE = self.initializeApi( protocol ) |
269 |
|
except Exception, ex: |
270 |
|
return self.updateReport('', '-1', str(ex)) |
271 |
|
|
272 |
|
# create remote dir |
273 |
< |
if protocol in ['gridftp','rfio']: |
273 |
> |
if Destination_SE.protocol in ['gridftp','rfio','srmv2']: |
274 |
|
try: |
275 |
< |
self.createDir( Destination_SE, protocol ) |
275 |
> |
self.createDir( Destination_SE, Destination_SE.protocol ) |
276 |
|
except Exception, ex: |
277 |
|
return self.updateReport('', '60316', str(ex)) |
278 |
|
|
282 |
|
sbi_dest = SBinterface(Destination_SE) |
283 |
|
sbi_source = SBinterface(Source_SE) |
284 |
|
except ProtocolMismatch, ex: |
285 |
< |
msg = str(ex)+'\n' |
286 |
< |
msg += "ERROR : Unable to create SBinterface with %s protocol\n"%protocol |
287 |
< |
return self.updateReport('', '-1', str(ex)) |
285 |
> |
msg = "ERROR : Unable to create SBinterface with %s protocol"%protocol |
286 |
> |
msg += str(ex) |
287 |
> |
return self.updateReport('', '-1', msg) |
288 |
|
|
289 |
|
results = {} |
290 |
|
## loop over the complete list of files |
291 |
|
for filetocopy in list_file: |
292 |
< |
if self.debug : print 'start real copy for %s'%filetocopy |
292 |
> |
if self.debug : print '\tStart real copy for %s'%filetocopy |
293 |
|
try : |
294 |
< |
ErCode, msg = self.checkFileExist( sbi_source, sbi_dest, filetocopy ) |
294 |
> |
ErCode, msg = self.checkFileExist( sbi_source, sbi_dest, filetocopy, options ) |
295 |
|
except Exception, ex: |
296 |
|
ErCode = -1 |
297 |
|
msg = str(ex) |
298 |
|
if ErCode == '0': |
299 |
|
ErCode, msg = self.makeCopy( sbi, filetocopy , options, protocol,sbi_dest ) |
300 |
< |
if self.debug : print 'Copy results for %s is %s'%( os.path.basename(filetocopy), ErCode) |
300 |
> |
if self.debug : print '\tCopy results for %s is %s'%( os.path.basename(filetocopy), ErCode) |
301 |
|
results.update( self.updateReport(filetocopy, ErCode, msg)) |
302 |
|
return results |
303 |
|
|
306 |
|
""" |
307 |
|
Create the storage interface. |
308 |
|
""" |
309 |
+ |
if self.debug : print 'storageInterface():\n' |
310 |
|
try: |
311 |
|
interface = SElement( FullPath(endpoint), protocol ) |
312 |
|
except ProtocolUnknown, ex: |
313 |
< |
msg = '' |
314 |
< |
if self.debug : msg = str(ex)+'\n' |
225 |
< |
msg += "ERROR : Unable to create interface with %s protocol\n"%protocol |
313 |
> |
msg = "ERROR : Unable to create interface with %s protocol"%protocol |
314 |
> |
msg += str(ex) |
315 |
|
raise Exception(msg) |
316 |
|
|
317 |
|
return interface |
321 |
|
Create remote dir for gsiftp REALLY TEMPORARY |
322 |
|
this should be transparent at SE API level. |
323 |
|
""" |
324 |
+ |
if self.debug : print 'createDir():\n' |
325 |
|
msg = '' |
326 |
|
try: |
327 |
|
action = SBinterface( Destination_SE ) |
328 |
|
action.createDir() |
329 |
< |
if self.debug: msg+= "The directory has been created using protocol %s\n"%protocol |
329 |
> |
if self.debug: print "\tThe directory has been created using protocol %s"%protocol |
330 |
|
except TransferException, ex: |
331 |
< |
msg = str(ex) |
331 |
> |
msg = "ERROR: problem with the directory creation using %s protocol "%protocol |
332 |
> |
msg += str(ex) |
333 |
|
if self.debug : |
334 |
< |
msg += str(ex.detail)+'\n' |
335 |
< |
msg += str(ex.output)+'\n' |
336 |
< |
msg += "ERROR: problem with the directory creation using %s protocol \n"%protocol |
337 |
< |
raise Exceptions(msg) |
334 |
> |
dbgmsg = '\t'+msg+'\n\t'+str(ex.detail)+'\n' |
335 |
> |
dbgmsg += '\t'+str(ex.output)+'\n' |
336 |
> |
print dbgmsg |
337 |
> |
raise Exception(msg) |
338 |
|
except OperationException, ex: |
339 |
< |
msg = str(ex) |
340 |
< |
if self.debug : msg += str(ex.detail)+'\n' |
341 |
< |
msg += "ERROR: problem with the directory creation using %s protocol \n"%protocol |
342 |
< |
|
339 |
> |
msg = "ERROR: problem with the directory creation using %s protocol "%protocol |
340 |
> |
msg += str(ex) |
341 |
> |
if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n' |
342 |
> |
raise Exception(msg) |
343 |
> |
except MissingDestination, ex: |
344 |
> |
msg = "ERROR: problem with the directory creation using %s protocol "%protocol |
345 |
> |
msg += str(ex) |
346 |
> |
if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n' |
347 |
> |
raise Exception(msg) |
348 |
> |
except AlreadyExistsException, ex: |
349 |
> |
if self.debug: print "\tThe directory already exist" |
350 |
> |
pass |
351 |
|
return msg |
352 |
|
|
353 |
< |
def checkFileExist( self, sbi_source, sbi_dest, filetocopy ): |
353 |
> |
def checkFileExist( self, sbi_source, sbi_dest, filetocopy, option ): |
354 |
|
""" |
355 |
|
Check both if source file exist AND |
356 |
|
if destination file ALREADY exist. |
357 |
|
""" |
358 |
+ |
if self.debug : print 'checkFileExist():\n' |
359 |
|
ErCode = '0' |
360 |
|
msg = '' |
361 |
|
f_tocopy=filetocopy |
362 |
|
if self.source_prot != 'local':f_tocopy = os.path.basename(filetocopy) |
363 |
|
try: |
364 |
< |
checkSource = sbi_source.checkExists( f_tocopy ) |
364 |
> |
checkSource = sbi_source.checkExists( f_tocopy , opt=option ) |
365 |
> |
if self.debug : print '\tCheck for local file %s exist succeded \n'%f_tocopy |
366 |
|
except OperationException, ex: |
367 |
< |
msg = str(ex) |
367 |
> |
msg ='ERROR: problems checkig if source file %s exist'%filetocopy |
368 |
> |
msg += str(ex) |
369 |
|
if self.debug : |
370 |
< |
msg += str(ex.detail)+'\n' |
371 |
< |
msg += str(ex.output)+'\n' |
372 |
< |
msg +='ERROR: problems checkig if source file %s exist'%filetocopy |
370 |
> |
dbgmsg = '\t'+msg+'\n\t'+str(ex.detail)+'\n' |
371 |
> |
dbgmsg += '\t'+str(ex.output)+'\n' |
372 |
> |
print dbgmsg |
373 |
|
raise Exception(msg) |
374 |
|
except WrongOption, ex: |
375 |
< |
msg = str(ex) |
375 |
> |
msg ='ERROR problems checkig if source file % exist'%filetocopy |
376 |
> |
msg += str(ex) |
377 |
|
if self.debug : |
378 |
< |
msg += str(ex.detail)+'\n' |
379 |
< |
msg += str(ex.output)+'\n' |
380 |
< |
msg +='ERROR problems checkig if source file % exist'%filetocopy |
378 |
> |
dbgmsg = '\t'+msg+'\n\t'+str(ex.detail)+'\n' |
379 |
> |
dbgmsg += '\t'+str(ex.output)+'\n' |
380 |
> |
print dbgmsg |
381 |
> |
raise Exception(msg) |
382 |
> |
except MissingDestination, ex: |
383 |
> |
msg ='ERROR problems checkig if source file % exist'%filetocopy |
384 |
> |
msg += str(ex) |
385 |
> |
if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n' |
386 |
|
raise Exception(msg) |
387 |
|
if not checkSource : |
388 |
|
ErCode = '60302' |
389 |
|
msg = "ERROR file %s do not exist"%os.path.basename(filetocopy) |
390 |
|
return ErCode, msg |
283 |
– |
|
391 |
|
f_tocopy=filetocopy |
392 |
|
if self.dest_prot != 'local':f_tocopy = os.path.basename(filetocopy) |
393 |
|
try: |
394 |
< |
check = sbi_dest.checkExists( f_tocopy ) |
394 |
> |
check = sbi_dest.checkExists( f_tocopy, opt=option ) |
395 |
> |
if self.debug : print '\tCheck for remote file %s exist succeded \n'%f_tocopy |
396 |
|
except OperationException, ex: |
397 |
< |
msg = str(ex) |
397 |
> |
msg = 'ERROR: problems checkig if file %s already exist'%filetocopy |
398 |
> |
msg += str(ex) |
399 |
|
if self.debug : |
400 |
< |
msg += str(ex.detail)+'\n' |
401 |
< |
msg += str(ex.output)+'\n' |
402 |
< |
msg +='ERROR: problems checkig if file %s already exist'%filetocopy |
400 |
> |
dbgmsg = '\t'+msg+'\n\t'+str(ex.detail)+'\n' |
401 |
> |
dbgmsg += '\t'+str(ex.output)+'\n' |
402 |
> |
print dbgmsg |
403 |
|
raise Exception(msg) |
404 |
|
except WrongOption, ex: |
405 |
< |
msg = str(ex) |
405 |
> |
msg = 'ERROR problems checkig if file % already exist'%filetocopy |
406 |
> |
msg += str(ex) |
407 |
|
if self.debug : |
408 |
< |
msg += str(ex.detail)+'\n' |
409 |
< |
msg += str(ex.output)+'\n' |
410 |
< |
msg +='ERROR problems checkig if file % already exist'%filetocopy |
408 |
> |
msg += '\t'+msg+'\n\t'+str(ex.detail)+'\n' |
409 |
> |
msg += '\t'+str(ex.output)+'\n' |
410 |
> |
raise Exception(msg) |
411 |
> |
except MissingDestination, ex: |
412 |
> |
msg ='ERROR problems checkig if source file % exist'%filetocopy |
413 |
> |
msg += str(ex) |
414 |
> |
if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n' |
415 |
|
raise Exception(msg) |
416 |
|
if check : |
417 |
|
ErCode = '60303' |
423 |
|
""" |
424 |
|
call the copy API. |
425 |
|
""" |
426 |
+ |
if self.debug : print 'makeCopy():\n' |
427 |
|
path = os.path.dirname(filetocopy) |
428 |
|
file_name = os.path.basename(filetocopy) |
429 |
|
source_file = filetocopy |
431 |
|
if self.params['source'] == '' and path == '': |
432 |
|
source_file = os.path.abspath(filetocopy) |
433 |
|
elif self.params['destination'] =='': |
434 |
< |
dest_file = os.path.join(os.getcwd(),file_name) |
434 |
> |
destDir = self.params.get('destinationDir',os.getcwd()) |
435 |
> |
dest_file = os.path.join(destDir,file_name) |
436 |
|
elif self.params['source'] != '' and self.params['destination'] != '' : |
437 |
|
source_file = file_name |
438 |
|
|
442 |
|
try: |
443 |
|
sbi.copy( source_file , dest_file , opt = option) |
444 |
|
except TransferException, ex: |
445 |
< |
msg = str(ex) |
445 |
> |
msg = "Problem copying %s file" % filetocopy |
446 |
> |
msg += str(ex) |
447 |
|
if self.debug : |
448 |
< |
msg += str(ex.detail)+'\n' |
449 |
< |
msg += str(ex.output)+'\n' |
450 |
< |
msg += "Problem copying %s file" % filetocopy |
448 |
> |
dbgmsg = '\t'+msg+'\n\t'+str(ex.detail)+'\n' |
449 |
> |
dbgmsg += '\t'+str(ex.output)+'\n' |
450 |
> |
print dbgmsg |
451 |
|
ErCode = '60307' |
452 |
|
except WrongOption, ex: |
453 |
< |
msg = str(ex) |
453 |
> |
msg = "Problem copying %s file" % filetocopy |
454 |
> |
msg += str(ex) |
455 |
|
if self.debug : |
456 |
< |
msg += str(ex.detail)+'\n' |
457 |
< |
msg += str(ex.output)+'\n' |
458 |
< |
msg += "Problem copying %s file" % filetocopy |
456 |
> |
dbgmsg = '\t'+msg+'\n\t'+str(ex.detail)+'\n' |
457 |
> |
dbgmsg += '\t'+str(ex.output)+'\n' |
458 |
> |
print dbsmsg |
459 |
|
ErCode = '60307' |
460 |
< |
if ErCode == '0' and protocol.find('srm') == 0: |
460 |
> |
if ErCode == '0' and protocol.find('srmv') == 0: |
461 |
|
remote_file_size = -1 |
462 |
|
local_file_size = os.path.getsize( source_file ) |
463 |
|
try: |
464 |
< |
remote_file_size = sbi_dest.getSize( dest_file ) |
464 |
> |
remote_file_size = sbi_dest.getSize( dest_file, opt=option ) |
465 |
> |
if self.debug : print '\t Check of remote size succeded for file %s\n'%dest_file |
466 |
|
except TransferException, ex: |
467 |
< |
msg = str(ex) |
467 |
> |
msg = "Problem checking the size of %s file" % filetocopy |
468 |
> |
msg += str(ex) |
469 |
|
if self.debug : |
470 |
< |
msg += str(ex.detail)+'\n' |
471 |
< |
msg += str(ex.output)+'\n' |
472 |
< |
msg += "Problem checking the size of %s file" % filetocopy |
470 |
> |
dbgmsg = '\t'+msg+'\n\t'+str(ex.detail)+'\n' |
471 |
> |
dbgmsg += '\t'+str(ex.output)+'\n' |
472 |
> |
print dbgmsg |
473 |
|
ErCode = '60307' |
474 |
|
except WrongOption, ex: |
475 |
< |
msg = str(ex) |
475 |
> |
msg = "Problem checking the size of %s file" % filetocopy |
476 |
> |
msg += str(ex) |
477 |
|
if self.debug : |
478 |
< |
msg += str(ex.detail)+'\n' |
479 |
< |
msg += str(ex.output)+'\n' |
480 |
< |
msg += "Problem checking the size of %s file" % filetocopy |
478 |
> |
dbgmsg = '\t'+msg+'\n\t'+str(ex.detail)+'\n' |
479 |
> |
dbgmsg += '\t'+str(ex.output)+'\n' |
480 |
> |
print dbgmsg |
481 |
|
ErCode = '60307' |
482 |
|
if local_file_size != remote_file_size: |
483 |
|
msg = "File size dosn't match: local size = %s ; remote size = %s " % (local_file_size, remote_file_size) |
484 |
|
ErCode = '60307' |
485 |
|
|
486 |
+ |
if ErCode != '0': |
487 |
+ |
try : |
488 |
+ |
self.removeFile( sbi_dest, dest_file, option ) |
489 |
+ |
except Exception, ex: |
490 |
+ |
msg += '\n'+str(ex) |
491 |
|
return ErCode, msg |
492 |
|
|
493 |
+ |
def removeFile( self, sbi_dest, filetocopy, option ): |
494 |
+ |
""" |
495 |
+ |
""" |
496 |
+ |
if self.debug : print 'removeFile():\n' |
497 |
+ |
f_tocopy=filetocopy |
498 |
+ |
if self.dest_prot != 'local':f_tocopy = os.path.basename(filetocopy) |
499 |
+ |
try: |
500 |
+ |
sbi_dest.delete( f_tocopy, opt=option ) |
501 |
+ |
if self.debug : '\t deletion of file %s succeeded\n'%str(filetocopy) |
502 |
+ |
except OperationException, ex: |
503 |
+ |
msg ='ERROR: problems removing partially staged file %s'%filetocopy |
504 |
+ |
msg += str(ex) |
505 |
+ |
if self.debug : |
506 |
+ |
dbgmsg = '\t'+msg+'\n\t'+str(ex.detail)+'\n' |
507 |
+ |
dbgmsg += '\t'+str(ex.output)+'\n' |
508 |
+ |
print dbgmsg |
509 |
+ |
raise Exception(msg) |
510 |
+ |
|
511 |
+ |
return |
512 |
+ |
|
513 |
|
def backup(self): |
514 |
|
""" |
515 |
|
Check infos from TFC using existing api obtaining: |
557 |
|
cmscp_exit_status = dict['erCode'] |
558 |
|
cmscp_exit_status = dict['erCode'] |
559 |
|
else: |
560 |
+ |
txt += 'echo "StageOutExitStatusReason ='+dict['reason']+'" | tee -a $RUNTIME_AREA/$repo\n' |
561 |
|
cmscp_exit_status = dict['erCode'] |
562 |
|
cmscp_exit_status = dict['erCode'] |
563 |
|
txt += '\n' |
606 |
|
import getopt |
607 |
|
|
608 |
|
allowedOpt = ["source=", "destination=", "inputFileList=", "outputFileList=", \ |
609 |
< |
"protocol=","option=", "middleware=", "srm_version=", "debug", "help"] |
609 |
> |
"protocol=","option=", "middleware=", "srm_version=", \ |
610 |
> |
"destinationDir=","debug", "help"] |
611 |
|
try: |
612 |
|
opts, args = getopt.getopt( sys.argv[1:], "", allowedOpt ) |
613 |
|
except getopt.GetoptError, err: |