ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/cmscp.py
Revision: 1.52
Committed: Wed May 27 17:40:12 2009 UTC (15 years, 11 months ago) by spiga
Content type: text/x-python
Branch: MAIN
CVS Tags: CRAB_2_6_0_pre6, CRAB_2_6_0_pre5, CRAB_2_6_0_pre4
Changes since 1.51: +4 -2 lines
Log Message:
fix for dashboard reporting

File Contents

# User Rev Content
1 spiga 1.51 #!/usr/bin/env python
2 mcinquil 1.16 import sys, os
3 ewv 1.7 from ProdCommon.Storage.SEAPI.SElement import SElement, FullPath
4 spiga 1.1 from ProdCommon.Storage.SEAPI.SBinterface import *
5 spiga 1.11 from ProdCommon.Storage.SEAPI.Exceptions import *
6 spiga 1.1
7    
8     class cmscp:
9 spiga 1.8 def __init__(self, args):
10 spiga 1.1 """
11     cmscp
12 spiga 1.46 safe copy of local file to/from remote SE via lcg_cp/srmcp,
13 spiga 1.1 including success checking version also for CAF using rfcp command to copy the output to SE
14     input:
15     $1 middleware (CAF, LSF, LCG, OSG)
16 spiga 1.2 $2 local file (the absolute path of output file or just the name if it's in top dir)
17     $3 if needed: file name (the output file name)
18     $5 remote SE (complete endpoint)
19 ewv 1.7 $6 srm version
20 fanzago 1.38 --lfn $LFNBaseName
21 spiga 1.1 output:
22     return 0 if all ok
23     return 60307 if srmcp failed
24     return 60303 if file already exists in the SE
25     """
26 spiga 1.8
27 spiga 1.1 #set default
28 spiga 1.24 self.params = {"source":'', "destination":'','destinationDir':'', "inputFileList":'', "outputFileList":'', \
29 fanzago 1.38 "protocol":'', "option":'', "middleware":'', "srm_version":'srmv2', "lfn":'' }
30 ewv 1.14 self.debug = 0
31 spiga 1.46 self.local_stage = 0
32 spiga 1.8 self.params.update( args )
33 ewv 1.7
34 spiga 1.1 return
35    
36 spiga 1.8 def processOptions( self ):
37 spiga 1.1 """
38 spiga 1.8 check command line parameter
39 spiga 1.1 """
40 ewv 1.14 if 'help' in self.params.keys(): HelpOptions()
41     if 'debug' in self.params.keys(): self.debug = 1
42 spiga 1.45 if 'local_stage' in self.params.keys(): self.local_stage = 1
43 ewv 1.7
44     # source and dest cannot be undefined at same time
45 spiga 1.8 if not self.params['source'] and not self.params['destination'] :
46 spiga 1.36 HelpOptions()
47 ewv 1.7 # if middleware is not defined --> protocol cannot be empty
48 spiga 1.8 if not self.params['middleware'] and not self.params['protocol'] :
49     HelpOptions()
50    
51 ewv 1.7 # input file must be defined
52 spiga 1.35 if not self.params['inputFileList'] :
53 spiga 1.36 HelpOptions()
54 spiga 1.1 else:
55 spiga 1.8 file_to_copy=[]
56 ewv 1.14 if self.params['inputFileList'].find(','):
57 spiga 1.8 [file_to_copy.append(x.strip()) for x in self.params['inputFileList'].split(',')]
58 ewv 1.7 else:
59 spiga 1.8 file_to_copy.append(self.params['inputFileList'])
60     self.params['inputFileList'] = file_to_copy
61 ewv 1.7
62 spiga 1.46 if not self.params['lfn'] and self.local_stage == 1 : HelpOptions()
63 fanzago 1.38
64 spiga 1.1 ## TO DO:
65     #### add check for outFiles
66     #### add map {'inFileNAME':'outFileNAME'} to change out name
67    
68    
69 ewv 1.7 def run( self ):
70 spiga 1.1 """
71 ewv 1.7 Check if running on UI (no $middleware) or
72     on WN (on the Grid), and take different action
73 spiga 1.1 """
74 mcinquil 1.37 self.processOptions()
75 spiga 1.30 if self.debug: print 'calling run() : \n'
76 spiga 1.8 # stage out from WN
77     if self.params['middleware'] :
78 spiga 1.36 results = self.stager(self.params['middleware'],self.params['inputFileList'])
79 spiga 1.35 self.finalReport(results)
80 spiga 1.8 # Local interaction with SE
81 spiga 1.1 else:
82 spiga 1.36 results = self.copy(self.params['inputFileList'], self.params['protocol'], self.params['option'] )
83 spiga 1.35 return results
84 spiga 1.1
85 spiga 1.8 def setProtocol( self, middleware ):
86 spiga 1.1 """
87     define the allowed potocols based on $middlware
88 ewv 1.7 which depend on scheduler
89 spiga 1.1 """
90 spiga 1.8 # default To be used with "middleware"
91 fanzago 1.38 if self.debug:
92     print 'setProtocol() :\n'
93     print '\tmiddleware = %s utils \n'%middleware
94    
95 spiga 1.13 lcgOpt={'srmv1':'-b -D srmv1 -t 2400 --verbose',
96     'srmv2':'-b -D srmv2 -t 2400 --verbose'}
97     srmOpt={'srmv1':' -report ./srmcp.report -retry_timeout 480000 -retry_num 3 -streams_num=1 ',
98 ewv 1.33 'srmv2':' -report=./srmcp.report -retry_timeout=480000 -retry_num=3 '}
99 spiga 1.8 rfioOpt=''
100    
101 ewv 1.14 supported_protocol = None
102 spiga 1.40 if middleware.lower() in ['osg','lcg','condor','sge']:
103 spiga 1.12 supported_protocol = [('srm-lcg',lcgOpt[self.params['srm_version']]),\
104 spiga 1.30 (self.params['srm_version'],srmOpt[self.params['srm_version']])]
105 spiga 1.8 elif middleware.lower() in ['lsf','caf']:
106 ewv 1.14 supported_protocol = [('rfio',rfioOpt)]
107 spiga 1.1 else:
108 ewv 1.7 ## here we can add support for any kind of protocol,
109 spiga 1.1 ## maybe some local schedulers need something dedicated
110     pass
111     return supported_protocol
112 spiga 1.28
113 fanzago 1.41
114     def checkCopy (self, copy_results, len_list_files, prot, lfn='', se=''):
115     """
116     Checks the status of copy and update result dictionary
117 spiga 1.28 """
118     list_retry = []
119 fanzago 1.41 list_not_existing = []
120 spiga 1.28 list_ok = []
121 fanzago 1.41
122     if self.debug:
123     print 'in checkCopy() :\n'
124     for file, dict in copy_results.iteritems():
125     er_code = dict['erCode']
126     if er_code == '0':
127     list_ok.append(file)
128     reason = 'Copy succedeed with %s utils'%prot
129     dict['reason'] = reason
130     elif er_code == '60302':
131     list_not_existing.append( file )
132 mcinquil 1.50 elif er_code == '10041':
133 fanzago 1.41 list_retry.append( file )
134 mcinquil 1.50 ## WHAT TO DO IN GENERAL FAILURE CONDITION
135     #else:
136     # list_retry.append( file )
137 fanzago 1.41
138     if self.debug:
139     print "\t file %s \n"%file
140     print "\t dict['erCode'] %s \n"%dict['erCode']
141     print "\t dict['reason'] %s \n"%dict['reason']
142    
143     if (lfn != '') and (se != ''):
144     upDict = self.updateReport(file, er_code, dict['reason'], lfn, se)
145 spiga 1.28 else:
146 fanzago 1.41 upDict = self.updateReport(file, er_code, dict['reason'])
147    
148     copy_results.update(upDict)
149    
150     msg = ''
151     if len(list_ok) != 0:
152     msg += '\tCopy of %s succedeed with %s utils\n'%(str(list_ok),prot)
153     if len(list_ok) != len_list_files :
154     msg += '\tCopy of %s failed using %s for files \n'%(str(list_retry),prot)
155     msg += '\tCopy of %s failed using %s : files not found \n'%(str(list_not_existing),prot)
156     if self.debug : print msg
157    
158     return copy_results, list_ok, list_retry
159    
160 spiga 1.45 def LocalCopy(self, list_retry, results):
161 fanzago 1.41 """
162 spiga 1.45 Tries the stage out to the CloseSE
163 fanzago 1.38 """
164 fanzago 1.41 if self.debug:
165 spiga 1.45 print 'in LocalCopy() :\n'
166 fanzago 1.41 print '\t list_retry %s utils \n'%list_retry
167     print '\t len(list_retry) %s \n'%len(list_retry)
168    
169     list_files = list_retry
170     self.params['inputFilesList']=list_files
171    
172     ### copy backup
173     from ProdCommon.FwkJobRep.SiteLocalConfig import loadSiteLocalConfig
174     siteCfg = loadSiteLocalConfig()
175     seName = siteCfg.localStageOut.get("se-name", None)
176     catalog = siteCfg.localStageOut.get("catalog", None)
177     implName = siteCfg.localStageOut.get("command", None)
178     if (implName == 'srm'):
179     implName='srmv1'
180     self.params['srm_version']=implName
181     ##### to be improved ###############
182     if (implName == 'rfcp'):
183     self.params['middleware']='lsf'
184     ####################################
185    
186     self.params['protocol']=implName
187     tfc = siteCfg.trivialFileCatalog()
188    
189     if self.debug:
190     print '\t siteCFG %s \n'%siteCfg
191     print '\t seName %s \n'%seName
192     print '\t catalog %s \n'%catalog
193     print "\t self.params['protocol'] %s \n"%self.params['protocol']
194     print '\t tfc %s '%tfc
195     print "\t self.params['inputFilesList'] %s \n"%self.params['inputFilesList']
196    
197     file_backup=[]
198     for input in self.params['inputFilesList']:
199     file = self.params['lfn'] + os.path.basename(input)
200     surl = tfc.matchLFN(tfc.preferredProtocol, file)
201     file_backup.append(surl)
202     if self.debug:
203     print '\t lfn %s \n'%self.params['lfn']
204     print '\t file %s \n'%file
205     print '\t surl %s \n'%surl
206    
207     destination=os.path.dirname(file_backup[0])
208     self.params['destination']=destination
209    
210     if self.debug:
211     print "\t self.params['destination']%s \n"%self.params['destination']
212     print "\t self.params['protocol'] %s \n"%self.params['protocol']
213     print "\t self.params['option']%s \n"%self.params['option']
214    
215     for prot, opt in self.setProtocol( self.params['middleware'] ):
216 spiga 1.45 if self.debug: print '\tIn LocalCopy trying the stage out with %s utils \n'%prot
217     localCopy_results = self.copy( self.params['inputFileList'], prot, opt )
218     if localCopy_results.keys() == [''] or localCopy_results.keys() == '' :
219     results.update(localCopy_results)
220 fanzago 1.41 else:
221 spiga 1.45 localCopy_results, list_ok, list_retry = self.checkCopy(localCopy_results, len(list_files), prot, self.params['lfn'], seName)
222     results.update(localCopy_results)
223 fanzago 1.41 if len(list_ok) == len(list_files) :
224     break
225     if len(list_retry):
226     list_files = list_retry
227     else: break
228     if self.debug:
229 spiga 1.45 print "\t localCopy_results = %s \n"%localCopy_results
230 spiga 1.28
231 fanzago 1.41 return results
232    
233 spiga 1.8 def stager( self, middleware, list_files ):
234 spiga 1.1 """
235     Implement the logic for remote stage out
236     """
237 spiga 1.30
238 fanzago 1.38 if self.debug:
239     print 'stager() :\n'
240     print '\tmiddleware %s\n'%middleware
241 spiga 1.45 print '\tlist_files %s\n'%list_files
242 fanzago 1.38
243 spiga 1.6 results={}
244 spiga 1.8 for prot, opt in self.setProtocol( middleware ):
245 spiga 1.30 if self.debug: print '\tTrying the stage out with %s utils \n'%prot
246 spiga 1.8 copy_results = self.copy( list_files, prot, opt )
247 spiga 1.30 if copy_results.keys() == [''] or copy_results.keys() == '' :
248 spiga 1.13 results.update(copy_results)
249     else:
250 fanzago 1.41 copy_results, list_ok, list_retry = self.checkCopy(copy_results, len(list_files), prot)
251 spiga 1.13 results.update(copy_results)
252     if len(list_ok) == len(list_files) :
253     break
254 fanzago 1.41 if len(list_retry):
255     list_files = list_retry
256     else: break
257    
258 spiga 1.45 if self.local_stage:
259 fanzago 1.43 if len(list_retry):
260 spiga 1.46 results = self.LocalCopy(list_retry, results)
261 fanzago 1.38
262     if self.debug:
263     print "\t results %s \n"%results
264 spiga 1.1 return results
265    
266     def initializeApi(self, protocol ):
267     """
268 ewv 1.7 Instantiate storage interface
269 spiga 1.1 """
270 spiga 1.30 if self.debug : print 'initializeApi() :\n'
271 spiga 1.20 self.source_prot = protocol
272     self.dest_prot = protocol
273     if not self.params['source'] : self.source_prot = 'local'
274     Source_SE = self.storageInterface( self.params['source'], self.source_prot )
275     if not self.params['destination'] : self.dest_prot = 'local'
276     Destination_SE = self.storageInterface( self.params['destination'], self.dest_prot )
277 spiga 1.1
278     if self.debug :
279 spiga 1.30 msg = '\t(source=%s, protocol=%s)'%(self.params['source'], self.source_prot)
280     msg += '\t(destination=%s, protocol=%s)'%(self.params['destination'], self.dest_prot)
281 mcinquil 1.32 print msg
282 spiga 1.1
283     return Source_SE, Destination_SE
284    
285 spiga 1.8 def copy( self, list_file, protocol, options ):
286 spiga 1.1 """
287 ewv 1.7 Make the real file copy using SE API
288 spiga 1.1 """
289 mcinquil 1.37 msg = ""
290 spiga 1.1 if self.debug :
291 spiga 1.30 msg = 'copy() :\n'
292     msg += '\tusing %s protocol\n'%protocol
293     print msg
294 spiga 1.13 try:
295     Source_SE, Destination_SE = self.initializeApi( protocol )
296     except Exception, ex:
297     return self.updateReport('', '-1', str(ex))
298 ewv 1.14
299 ewv 1.7 # create remote dir
300 mcinquil 1.32 if Destination_SE.protocol in ['gridftp','rfio','srmv2']:
301 spiga 1.13 try:
302 mcinquil 1.32 self.createDir( Destination_SE, Destination_SE.protocol )
303 mcinquil 1.47 except OperationException, ex:
304 spiga 1.13 return self.updateReport('', '60316', str(ex))
305 mcinquil 1.50 ## when the client commands are not found (wrong env or really missing)
306     except MissingCommand, ex:
307     msg = "ERROR %s %s" %(str(ex), str(ex.detail))
308     return self.updateReport('', '10041', msg)
309 spiga 1.1
310     ## prepare for real copy ##
311 spiga 1.8 try :
312     sbi = SBinterface( Source_SE, Destination_SE )
313     sbi_dest = SBinterface(Destination_SE)
314 spiga 1.20 sbi_source = SBinterface(Source_SE)
315 spiga 1.11 except ProtocolMismatch, ex:
316 spiga 1.30 msg = "ERROR : Unable to create SBinterface with %s protocol"%protocol
317     msg += str(ex)
318     return self.updateReport('', '-1', msg)
319 spiga 1.1
320     results = {}
321 ewv 1.7 ## loop over the complete list of files
322     for filetocopy in list_file:
323 spiga 1.30 if self.debug : print '\tStart real copy for %s'%filetocopy
324 spiga 1.13 try :
325 spiga 1.34 ErCode, msg = self.checkFileExist( sbi_source, sbi_dest, filetocopy, options )
326 spiga 1.13 except Exception, ex:
327 spiga 1.18 ErCode = -1
328     msg = str(ex)
329 ewv 1.7 if ErCode == '0':
330 spiga 1.19 ErCode, msg = self.makeCopy( sbi, filetocopy , options, protocol,sbi_dest )
331 spiga 1.30 if self.debug : print '\tCopy results for %s is %s'%( os.path.basename(filetocopy), ErCode)
332 spiga 1.1 results.update( self.updateReport(filetocopy, ErCode, msg))
333     return results
334 ewv 1.7
335 spiga 1.1
336     def storageInterface( self, endpoint, protocol ):
337     """
338 ewv 1.7 Create the storage interface.
339 spiga 1.1 """
340 spiga 1.30 if self.debug : print 'storageInterface():\n'
341 spiga 1.1 try:
342     interface = SElement( FullPath(endpoint), protocol )
343 spiga 1.11 except ProtocolUnknown, ex:
344 spiga 1.30 msg = "ERROR : Unable to create interface with %s protocol"%protocol
345     msg += str(ex)
346 spiga 1.13 raise Exception(msg)
347 spiga 1.1
348     return interface
349    
350     def createDir(self, Destination_SE, protocol):
351     """
352 spiga 1.8 Create remote dir for gsiftp REALLY TEMPORARY
353 ewv 1.7 this should be transparent at SE API level.
354 spiga 1.1 """
355 spiga 1.30 if self.debug : print 'createDir():\n'
356 ewv 1.14 msg = ''
357 spiga 1.1 try:
358     action = SBinterface( Destination_SE )
359     action.createDir()
360 spiga 1.30 if self.debug: print "\tThe directory has been created using protocol %s"%protocol
361 spiga 1.11 except TransferException, ex:
362 spiga 1.30 msg = "ERROR: problem with the directory creation using %s protocol "%protocol
363     msg += str(ex)
364 spiga 1.11 if self.debug :
365 spiga 1.30 dbgmsg = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
366     dbgmsg += '\t'+str(ex.output)+'\n'
367     print dbgmsg
368 spiga 1.29 raise Exception(msg)
369 spiga 1.11 except OperationException, ex:
370 spiga 1.30 msg = "ERROR: problem with the directory creation using %s protocol "%protocol
371     msg += str(ex)
372     if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n'
373 spiga 1.29 raise Exception(msg)
374 spiga 1.31 except MissingDestination, ex:
375     msg = "ERROR: problem with the directory creation using %s protocol "%protocol
376     msg += str(ex)
377     if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n'
378     raise Exception(msg)
379     except AlreadyExistsException, ex:
380     if self.debug: print "\tThe directory already exist"
381     pass
382 spiga 1.13 return msg
383 spiga 1.1
384 spiga 1.34 def checkFileExist( self, sbi_source, sbi_dest, filetocopy, option ):
385 spiga 1.1 """
386 spiga 1.20 Check both if source file exist AND
387     if destination file ALREADY exist.
388 ewv 1.7 """
389 spiga 1.30 if self.debug : print 'checkFileExist():\n'
390 spiga 1.8 ErCode = '0'
391     msg = ''
392 spiga 1.20 f_tocopy=filetocopy
393     if self.source_prot != 'local':f_tocopy = os.path.basename(filetocopy)
394 spiga 1.1 try:
395 spiga 1.34 checkSource = sbi_source.checkExists( f_tocopy , opt=option )
396 spiga 1.30 if self.debug : print '\tCheck for local file %s exist succeded \n'%f_tocopy
397 spiga 1.20 except OperationException, ex:
398 spiga 1.30 msg ='ERROR: problems checkig if source file %s exist'%filetocopy
399     msg += str(ex)
400 spiga 1.20 if self.debug :
401 spiga 1.30 dbgmsg = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
402     dbgmsg += '\t'+str(ex.output)+'\n'
403     print dbgmsg
404 spiga 1.20 raise Exception(msg)
405     except WrongOption, ex:
406 spiga 1.30 msg ='ERROR problems checkig if source file % exist'%filetocopy
407     msg += str(ex)
408 spiga 1.20 if self.debug :
409 spiga 1.30 dbgmsg = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
410     dbgmsg += '\t'+str(ex.output)+'\n'
411     print dbgmsg
412 spiga 1.20 raise Exception(msg)
413 spiga 1.31 except MissingDestination, ex:
414     msg ='ERROR problems checkig if source file % exist'%filetocopy
415     msg += str(ex)
416     if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n'
417     raise Exception(msg)
418 mcinquil 1.50 ## when the client commands are not found (wrong env or really missing)
419     except MissingCommand, ex:
420     ErCode = '10041'
421     msg = "ERROR %s %s" %(str(ex), str(ex.detail))
422     return ErCode, msg
423 spiga 1.20 if not checkSource :
424     ErCode = '60302'
425     msg = "ERROR file %s do not exist"%os.path.basename(filetocopy)
426     return ErCode, msg
427     f_tocopy=filetocopy
428     if self.dest_prot != 'local':f_tocopy = os.path.basename(filetocopy)
429     try:
430 spiga 1.34 check = sbi_dest.checkExists( f_tocopy, opt=option )
431 spiga 1.30 if self.debug : print '\tCheck for remote file %s exist succeded \n'%f_tocopy
432 spiga 1.11 except OperationException, ex:
433 spiga 1.30 msg = 'ERROR: problems checkig if file %s already exist'%filetocopy
434     msg += str(ex)
435 spiga 1.11 if self.debug :
436 spiga 1.30 dbgmsg = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
437     dbgmsg += '\t'+str(ex.output)+'\n'
438     print dbgmsg
439 spiga 1.13 raise Exception(msg)
440 spiga 1.11 except WrongOption, ex:
441 spiga 1.30 msg = 'ERROR problems checkig if file % already exist'%filetocopy
442     msg += str(ex)
443 spiga 1.11 if self.debug :
444 spiga 1.30 msg += '\t'+msg+'\n\t'+str(ex.detail)+'\n'
445     msg += '\t'+str(ex.output)+'\n'
446 spiga 1.13 raise Exception(msg)
447 spiga 1.31 except MissingDestination, ex:
448     msg ='ERROR problems checkig if source file % exist'%filetocopy
449     msg += str(ex)
450     if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n'
451     raise Exception(msg)
452 mcinquil 1.50 ## when the client commands are not found (wrong env or really missing)
453     except MissingCommand, ex:
454     ErCode = '10041'
455     msg = "ERROR %s %s" %(str(ex), str(ex.detail))
456     return ErCode, msg
457 ewv 1.14 if check :
458 ewv 1.7 ErCode = '60303'
459 spiga 1.20 msg = "file %s already exist"%os.path.basename(filetocopy)
460 ewv 1.14
461 spiga 1.13 return ErCode, msg
462 spiga 1.1
463 spiga 1.19 def makeCopy(self, sbi, filetocopy, option, protocol, sbi_dest ):
464 spiga 1.1 """
465 ewv 1.7 call the copy API.
466 spiga 1.1 """
467 spiga 1.30 if self.debug : print 'makeCopy():\n'
468 ewv 1.7 path = os.path.dirname(filetocopy)
469 spiga 1.1 file_name = os.path.basename(filetocopy)
470     source_file = filetocopy
471     dest_file = file_name ## to be improved supporting changing file name TODO
472 spiga 1.8 if self.params['source'] == '' and path == '':
473 spiga 1.1 source_file = os.path.abspath(filetocopy)
474 spiga 1.8 elif self.params['destination'] =='':
475 spiga 1.24 destDir = self.params.get('destinationDir',os.getcwd())
476     dest_file = os.path.join(destDir,file_name)
477 spiga 1.8 elif self.params['source'] != '' and self.params['destination'] != '' :
478 ewv 1.7 source_file = file_name
479 spiga 1.8
480 spiga 1.1 ErCode = '0'
481     msg = ''
482 ewv 1.7
483 spiga 1.1 try:
484 spiga 1.8 sbi.copy( source_file , dest_file , opt = option)
485 spiga 1.11 except TransferException, ex:
486 spiga 1.30 msg = "Problem copying %s file" % filetocopy
487     msg += str(ex)
488 spiga 1.11 if self.debug :
489 spiga 1.30 dbgmsg = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
490     dbgmsg += '\t'+str(ex.output)+'\n'
491 mcinquil 1.32 print dbgmsg
492 spiga 1.1 ErCode = '60307'
493 spiga 1.11 except WrongOption, ex:
494 spiga 1.30 msg = "Problem copying %s file" % filetocopy
495     msg += str(ex)
496 spiga 1.11 if self.debug :
497 spiga 1.30 dbgmsg = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
498     dbgmsg += '\t'+str(ex.output)+'\n'
499 fanzago 1.48 print dbgmsg
500 spiga 1.44 except SizeZeroException, ex:
501     msg = "Problem copying %s file" % filetocopy
502     msg += str(ex)
503     if self.debug :
504     dbgmsg = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
505     dbgmsg += '\t'+str(ex.output)+'\n'
506 fanzago 1.48 print dbgmsg
507 spiga 1.13 ErCode = '60307'
508 mcinquil 1.50 ## when the client commands are not found (wrong env or really missing)
509     except MissingCommand, ex:
510     ErCode = '10041'
511     msg = "Problem copying %s file" % filetocopy
512     msg += str(ex)
513     if self.debug :
514     dbgmsg = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
515     dbgmsg += '\t'+str(ex.output)+'\n'
516     print dbgmsg
517 spiga 1.24 if ErCode == '0' and protocol.find('srmv') == 0:
518 spiga 1.19 remote_file_size = -1
519     local_file_size = os.path.getsize( source_file )
520     try:
521 spiga 1.34 remote_file_size = sbi_dest.getSize( dest_file, opt=option )
522 spiga 1.30 if self.debug : print '\t Check of remote size succeded for file %s\n'%dest_file
523 spiga 1.19 except TransferException, ex:
524 spiga 1.30 msg = "Problem checking the size of %s file" % filetocopy
525     msg += str(ex)
526 spiga 1.19 if self.debug :
527 spiga 1.30 dbgmsg = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
528     dbgmsg += '\t'+str(ex.output)+'\n'
529     print dbgmsg
530 spiga 1.19 ErCode = '60307'
531     except WrongOption, ex:
532 spiga 1.30 msg = "Problem checking the size of %s file" % filetocopy
533     msg += str(ex)
534 spiga 1.19 if self.debug :
535 spiga 1.30 dbgmsg = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
536     dbgmsg += '\t'+str(ex.output)+'\n'
537     print dbgmsg
538 spiga 1.19 ErCode = '60307'
539     if local_file_size != remote_file_size:
540     msg = "File size dosn't match: local size = %s ; remote size = %s " % (local_file_size, remote_file_size)
541     ErCode = '60307'
542 ewv 1.14
543 spiga 1.27 if ErCode != '0':
544     try :
545 spiga 1.34 self.removeFile( sbi_dest, dest_file, option )
546 spiga 1.27 except Exception, ex:
547     msg += '\n'+str(ex)
548 spiga 1.1 return ErCode, msg
549 ewv 1.7
550 spiga 1.34 def removeFile( self, sbi_dest, filetocopy, option ):
551 spiga 1.30 """
552     """
553     if self.debug : print 'removeFile():\n'
554 spiga 1.21 f_tocopy=filetocopy
555     if self.dest_prot != 'local':f_tocopy = os.path.basename(filetocopy)
556     try:
557 spiga 1.34 sbi_dest.delete( f_tocopy, opt=option )
558 spiga 1.30 if self.debug : '\t deletion of file %s succeeded\n'%str(filetocopy)
559 spiga 1.21 except OperationException, ex:
560 spiga 1.30 msg ='ERROR: problems removing partially staged file %s'%filetocopy
561     msg += str(ex)
562 spiga 1.21 if self.debug :
563 spiga 1.30 dbgmsg = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
564     dbgmsg += '\t'+str(ex.output)+'\n'
565     print dbgmsg
566 spiga 1.21 raise Exception(msg)
567    
568     return
569    
570 spiga 1.8 def updateReport(self, file, erCode, reason, lfn='', se='' ):
571     """
572     Update the final stage out infos
573     """
574     jobStageInfo={}
575     jobStageInfo['erCode']=erCode
576     jobStageInfo['reason']=reason
577     jobStageInfo['lfn']=lfn
578     jobStageInfo['se']=se
579 spiga 1.1
580 spiga 1.8 report = { file : jobStageInfo}
581     return report
582 ewv 1.7
583 spiga 1.8 def finalReport( self , results ):
584     """
585     It a list of LFNs for each SE where data are stored.
586     allow "crab -copyLocal" or better "crab -copyOutput". TO_DO.
587 spiga 1.1 """
588 spiga 1.8 outFile = open('cmscpReport.sh',"a")
589     cmscp_exit_status = 0
590     txt = ''
591 ewv 1.14 for file, dict in results.iteritems():
592 spiga 1.52 reason = str(dict['reason'])
593     if str(reason).find("'") > -1:
594     reason = " ".join(reason.split("'"))
595     reason="'%s'"%reason
596 ewv 1.14 if file:
597 spiga 1.11 if dict['lfn']=='':
598     lfn = '$LFNBaseName/'+os.path.basename(file)
599     se = '$SE'
600     else:
601 mcinquil 1.16 lfn = dict['lfn']+os.path.basename(file)
602 spiga 1.11 se = dict['se']
603     #dict['lfn'] # to be implemented
604 fanzago 1.39 txt += 'echo "Report for File: '+file+'"\n'
605     txt += 'echo "LFN: '+lfn+'"\n'
606     txt += 'echo "StorageElement: '+se+'"\n'
607 spiga 1.49 txt += 'echo "StageOutExitStatusReason = %s" | tee -a $RUNTIME_AREA/$repo\n'%reason
608 spiga 1.11 txt += 'echo "StageOutSE = '+se+'" >> $RUNTIME_AREA/$repo\n'
609 fanzago 1.42 #txt += 'export LFNBaseName='+lfn+'\n'
610 fanzago 1.39 txt += 'export SE='+se+'\n'
611 fanzago 1.41
612 spiga 1.11 if dict['erCode'] != '0':
613     cmscp_exit_status = dict['erCode']
614 spiga 1.12 else:
615 spiga 1.49 txt += 'echo "StageOutExitStatusReason = %s" | tee -a $RUNTIME_AREA/$repo\n'%reason
616 spiga 1.12 cmscp_exit_status = dict['erCode']
617     cmscp_exit_status = dict['erCode']
618 spiga 1.8 txt += '\n'
619     txt += 'export StageOutExitStatus='+str(cmscp_exit_status)+'\n'
620     txt += 'echo "StageOutExitStatus = '+str(cmscp_exit_status)+'" | tee -a $RUNTIME_AREA/$repo\n'
621     outFile.write(str(txt))
622     outFile.close()
623     return
624    
625    
626     def usage():
627    
628     msg="""
629 spiga 1.46 cmscp:
630     safe copy of local file to/from remote SE via lcg_cp/srmcp,
631     including success checking version also for CAF using rfcp command to copy the output to SE
632 spiga 1.8
633 spiga 1.46 accepted parameters:
634     source =
635     destination =
636     inputFileList =
637     outputFileList =
638     protocol =
639     option =
640     middleware =
641     srm_version =
642     destinationDir =
643     lfn= =
644     local_stage = activate stage fall back
645     debug = activate verbose print out
646     help = print on line man and exit
647    
648     mandatory:
649     * "source" and/or "destination" must always be defined
650     * either "middleware" or "protocol" must always be defined
651     * "inputFileList" must always be defined
652     * if "local_stage" = 1 also "lfn" must be defined
653 spiga 1.8 """
654 ewv 1.14 print msg
655 spiga 1.8
656 ewv 1.14 return
657 spiga 1.8
658     def HelpOptions(opts=[]):
659     """
660     Check otps, print help if needed
661 ewv 1.14 prepare dict = { opt : value }
662 spiga 1.8 """
663     dict_args = {}
664     if len(opts):
665     for opt, arg in opts:
666 ewv 1.14 dict_args[opt.split('--')[1]] = arg
667 spiga 1.8 if opt in ('-h','-help','--help') :
668     usage()
669     sys.exit(0)
670     return dict_args
671     else:
672     usage()
673     sys.exit(0)
674 spiga 1.1
675     if __name__ == '__main__' :
676 spiga 1.8
677 ewv 1.14 import getopt
678 spiga 1.8
679     allowedOpt = ["source=", "destination=", "inputFileList=", "outputFileList=", \
680 spiga 1.25 "protocol=","option=", "middleware=", "srm_version=", \
681 spiga 1.46 "destinationDir=", "lfn=", "local_stage", "debug", "help"]
682 ewv 1.14 try:
683     opts, args = getopt.getopt( sys.argv[1:], "", allowedOpt )
684 spiga 1.8 except getopt.GetoptError, err:
685     print err
686     HelpOptions()
687     sys.exit(2)
688 ewv 1.14
689 spiga 1.8 dictArgs = HelpOptions(opts)
690 spiga 1.1 try:
691 spiga 1.8 cmscp_ = cmscp(dictArgs)
692 spiga 1.1 cmscp_.run()
693 spiga 1.11 except Exception, ex :
694     print str(ex)
695 spiga 1.1