ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/cmscp.py

Comparing COMP/CRAB/python/cmscp.py (file contents):
Revision 1.20 by spiga, Tue Oct 28 17:40:55 2008 UTC vs.
Revision 1.56 by edelmann, Wed Jun 10 06:37:33 2009 UTC

#	Line 1 | Line 1
1		#!/usr/bin/env python
2	–
2		import sys, os
3		from ProdCommon.Storage.SEAPI.SElement import SElement, FullPath
4		from ProdCommon.Storage.SEAPI.SBinterface import *
#	Line 10 | Line 9 | class cmscp:
9		def __init__(self, args):
10		"""
11		cmscp
12	<
14	<	safe copy of local file in current directory to remote SE via lcg_cp/srmcp,
12	>	safe copy of local file  to/from remote SE via lcg_cp/srmcp,
13		including success checking  version also for CAF using rfcp command to copy the output to SE
14		input:
15		$1 middleware (CAF, LSF, LCG, OSG)
#	Line 19 | Line 17 | class cmscp:
17		$3 if needed: file name (the output file name)
18		$5 remote SE (complete endpoint)
19		$6 srm version
20	+	--lfn $LFNBaseName
21		output:
22		return 0 if all ok
23		return 60307 if srmcp failed
#	Line 26 | Line 25 | class cmscp:
25		"""
26
27		#set default
28	<	self.params = {"source":'', "destination":'', "inputFileList":'', "outputFileList":'', \
29	<	"protocol":'', "option":'', "middleware":'', "srm_version":'srmv2'}
28	>	self.params = {"source":'', "destination":'','destinationDir':'', "inputFileList":'', "outputFileList":'', \
29	>	"protocol":'', "option":'', "middleware":'', "srm_version":'srmv2', "lfn":'' }
30		self.debug = 0
31	<
31	>	self.local_stage = 0
32		self.params.update( args )
33
34		return
#	Line 38 | Line 37 | class cmscp:
37		"""
38		check command line parameter
39		"""
41	–
40		if 'help' in self.params.keys(): HelpOptions()
41		if 'debug' in self.params.keys(): self.debug = 1
42	+	if 'local_stage' in self.params.keys(): self.local_stage = 1
43
44		# source and dest cannot be undefined at same time
45		if not self.params['source']  and not self.params['destination'] :
46		HelpOptions()
48	–
47		# if middleware is not defined --> protocol cannot be empty
48		if not self.params['middleware'] and not self.params['protocol'] :
49		HelpOptions()
50
51		# input file must be defined
52	<	if not self.params['inputFileList'] : HelpOptions()
52	>	if not self.params['inputFileList'] :
53	>	HelpOptions()
54		else:
55		file_to_copy=[]
56		if self.params['inputFileList'].find(','):
#	Line 60 | Line 59 | class cmscp:
59		file_to_copy.append(self.params['inputFileList'])
60		self.params['inputFileList'] = file_to_copy
61
62	+	if not self.params['lfn'] and self.local_stage == 1 : HelpOptions()
63	+
64		## TO DO:
65		#### add check for outFiles
66		#### add map {'inFileNAME':'outFileNAME'} to change out name
67
67	–	return
68
69		def run( self ):
70		"""
71		Check if running on UI (no $middleware) or
72		on WN (on the Grid), and take different action
73		"""
74	–
74		self.processOptions()
75	+	if self.debug: print 'calling run() : \n'
76		# stage out from WN
77		if self.params['middleware'] :
78	<	results = self.stager(self.params['middleware'],self.params['inputFileList'])
79	<	self.finalReport(results)
78	>	results = self.stager(self.params['middleware'],self.params['inputFileList'])
79	>	self.finalReport(results)
80		# Local interaction with SE
81		else:
82	<	results = self.copy(self.params['inputFilesList'], self.params['protocol'], self.params['option'] )
83	<	return results
82	>	results = self.copy(self.params['inputFileList'], self.params['protocol'], self.params['option'] )
83	>	return results
84
85		def setProtocol( self, middleware ):
86		"""
#	Line 88 | Line 88 | class cmscp:
88		which depend on scheduler
89		"""
90		# default To be used with "middleware"
91	+	if self.debug:
92	+	print 'setProtocol() :\n'
93	+	print '\tmiddleware =  %s utils \n'%middleware
94	+
95		lcgOpt={'srmv1':'-b -D srmv1  -t 2400 --verbose',
96		'srmv2':'-b -D srmv2  -t 2400 --verbose'}
97		srmOpt={'srmv1':' -report ./srmcp.report -retry_timeout 480000 -retry_num 3 -streams_num=1 ',
98	<	'srmv2':' -report ./srmcp.report -retry_timeout 480000 -retry_num 3 '}
98	>	'srmv2':' -report=./srmcp.report -retry_timeout=480000 -retry_num=3 '}
99		rfioOpt=''
100
101		supported_protocol = None
102	<	if middleware.lower() in ['osg','lcg','condor']:
102	>	if middleware.lower() in ['osg','lcg','condor','sge']:
103		supported_protocol = [('srm-lcg',lcgOpt[self.params['srm_version']]),\
104	<	(self.params['srm_version'],srmOpt[self.params['srm_version']])]
104	>	(self.params['srm_version'],srmOpt[self.params['srm_version']])]
105		elif middleware.lower() in ['lsf','caf']:
106		supported_protocol = [('rfio',rfioOpt)]
107	+	elif middleware.lower() in ['arc']:
108	+	supported_protocol = [('srmv2','-debug'),('srmv1','-debug')]
109		else:
110		## here we can add support for any kind of protocol,
111		## maybe some local schedulers need something dedicated
112		pass
113		return supported_protocol
114
115	+
116	+	def checkCopy (self, copy_results, len_list_files, prot, lfn='', se=''):
117	+	"""
118	+	Checks the status of copy and update result dictionary
119	+	"""
120	+	list_retry = []
121	+	list_retry_localSE = []
122	+	list_not_existing = []
123	+	list_ok = []
124	+
125	+	if self.debug:
126	+	print 'in checkCopy() :\n'
127	+	for file, dict in copy_results.iteritems():
128	+	er_code = dict['erCode']
129	+	if er_code == '0':
130	+	list_ok.append(file)
131	+	reason = 'Copy succedeed with %s utils'%prot
132	+	dict['reason'] = reason
133	+	elif er_code == '60302':
134	+	list_not_existing.append( file )
135	+	elif er_code == '10041':
136	+	list_retry.append( file )
137	+	## WHAT TO DO IN GENERAL FAILURE CONDITION
138	+	else:
139	+	list_retry_localSE.append( file )
140	+
141	+	if self.debug:
142	+	print "\t file %s \n"%file
143	+	print "\t dict['erCode'] %s \n"%dict['erCode']
144	+	print "\t dict['reason'] %s \n"%dict['reason']
145	+
146	+	if (lfn != '') and (se != ''):
147	+	upDict = self.updateReport(file, er_code, dict['reason'], lfn, se)
148	+	else:
149	+	upDict = self.updateReport(file, er_code, dict['reason'])
150	+
151	+	copy_results.update(upDict)
152	+
153	+	msg = ''
154	+	if len(list_ok) != 0:
155	+	msg += '\tCopy of %s succedeed with %s utils\n'%(str(list_ok),prot)
156	+	if len(list_ok) != len_list_files :
157	+	msg += '\tCopy of %s failed using %s for files \n'%(str(list_retry),prot)
158	+	msg += '\tCopy of %s failed using %s : files not found \n'%(str(list_not_existing),prot)
159	+	if self.debug : print msg
160	+
161	+	return copy_results, list_ok, list_retry, list_retry_localSE
162	+
163	+	def LocalCopy(self, list_retry, results):
164	+	"""
165	+	Tries the stage out to the CloseSE
166	+	"""
167	+	if self.debug:
168	+	print 'in LocalCopy() :\n'
169	+	print '\t list_retry %s utils \n'%list_retry
170	+	print '\t len(list_retry) %s \n'%len(list_retry)
171	+
172	+	list_files = list_retry
173	+	self.params['inputFilesList']=list_files
174	+
175	+	### copy backup
176	+	from ProdCommon.FwkJobRep.SiteLocalConfig import loadSiteLocalConfig
177	+	siteCfg = loadSiteLocalConfig()
178	+	seName = siteCfg.localStageOut.get("se-name", None)
179	+	catalog = siteCfg.localStageOut.get("catalog", None)
180	+	implName = siteCfg.localStageOut.get("command", None)
181	+	if (implName == 'srm'):
182	+	implName='srmv1'
183	+	self.params['srm_version']=implName
184	+	##### to be improved ###############
185	+	if (implName == 'rfcp'):
186	+	self.params['middleware']='lsf'
187	+	####################################
188	+
189	+	self.params['protocol']=implName
190	+	tfc = siteCfg.trivialFileCatalog()
191	+
192	+	if self.debug:
193	+	print '\t siteCFG %s \n'%siteCfg
194	+	print '\t seName %s \n'%seName
195	+	print '\t catalog %s \n'%catalog
196	+	print "\t self.params['protocol'] %s \n"%self.params['protocol']
197	+	print '\t tfc %s '%tfc
198	+	print "\t self.params['inputFilesList'] %s \n"%self.params['inputFilesList']
199	+
200	+	file_backup=[]
201	+	for input in self.params['inputFilesList']:
202	+	file = self.params['lfn'] + os.path.basename(input)
203	+	surl = tfc.matchLFN(tfc.preferredProtocol, file)
204	+	file_backup.append(surl)
205	+	if self.debug:
206	+	print '\t lfn %s \n'%self.params['lfn']
207	+	print '\t file %s \n'%file
208	+	print '\t surl %s \n'%surl
209	+
210	+	destination=os.path.dirname(file_backup[0])
211	+	self.params['destination']=destination
212	+
213	+	if self.debug:
214	+	print "\t self.params['destination']%s \n"%self.params['destination']
215	+	print "\t self.params['protocol'] %s \n"%self.params['protocol']
216	+	print "\t self.params['option']%s \n"%self.params['option']
217	+
218	+	for prot, opt in self.setProtocol( self.params['middleware'] ):
219	+	if self.debug: print '\tIn LocalCopy trying the stage out with %s utils \n'%prot
220	+	localCopy_results = self.copy( self.params['inputFileList'], prot, opt )
221	+	if localCopy_results.keys() == [''] or localCopy_results.keys() == '' :
222	+	results.update(localCopy_results)
223	+	else:
224	+	localCopy_results, list_ok, list_retry, list_retry_localSE = self.checkCopy(localCopy_results, len(list_files), prot, self.params['lfn'], seName)
225	+	results.update(localCopy_results)
226	+	if len(list_ok) == len(list_files) :
227	+	break
228	+	if len(list_retry):
229	+	list_files = list_retry
230	+	else: break
231	+	if self.debug:
232	+	print "\t localCopy_results = %s \n"%localCopy_results
233	+
234	+	return results
235	+
236		def stager( self, middleware, list_files ):
237		"""
238		Implement the logic for remote stage out
239		"""
240	+
241	+	if self.debug:
242	+	print 'stager() :\n'
243	+	print '\tmiddleware %s\n'%middleware
244	+	print '\tlist_files %s\n'%list_files
245	+
246		results={}
247		for prot, opt in self.setProtocol( middleware ):
248	<	if self.debug: print 'Trying stage out with %s utils \n'%prot
248	>	if self.debug: print '\tTrying the stage out with %s utils \n'%prot
249		copy_results = self.copy( list_files, prot, opt )
250	<	list_retry = []
118	<	list_existing = []
119	<	list_ok = []
120	<	if copy_results.keys() == '':
250	>	if copy_results.keys() == [''] or copy_results.keys() == '' :
251		results.update(copy_results)
252		else:
253	<	for file, dict in copy_results.iteritems():
124	<	er_code = dict['erCode']
125	<	if er_code == '0':
126	<	list_ok.append(file)
127	<	reason = 'Copy succedeed with %s utils'%prot
128	<	upDict = self.updateReport(file, er_code, reason)
129	<	copy_results.update(upDict)
130	<	elif er_code == '60303': list_existing.append( file )
131	<	else: list_retry.append( file )
253	>	copy_results, list_ok, list_retry, list_retry_localSE = self.checkCopy(copy_results, len(list_files), prot)
254		results.update(copy_results)
133	–	if len(list_ok) != 0:
134	–	msg = 'Copy of %s succedeed with %s utils\n'%(str(list_ok),prot)
135	–	if self.debug : print msg
255		if len(list_ok) == len(list_files) :
256		break
257	<	else:
258	<	if self.debug : print 'Copy of files %s failed using %s...\n'%(str(list_retry)+str(list_existing),prot)
259	<	if len(list_retry): list_files = list_retry
260	<	else: break
261	<
262	<	#### TODO Daniele
263	<	#check is something fails and created related dict
264	<	#      backup = self.analyzeResults(results)
265	<
266	<	#      if backup :
148	<	#          msg = 'WARNING: backup logic is under implementation\n'
149	<	#          #backupDict = self.backup()
150	<	#          ### NOTE: IT MUST RETURN a DICT contains also LFN and SE Name
151	<	#          results.update(backupDict)
152	<	#          print msg
257	>	if len(list_retry):
258	>	list_files = list_retry
259	>	else: break
260	>
261	>	if self.local_stage:
262	>	if len(list_retry_localSE):
263	>	results = self.LocalCopy(list_retry_localSE, results)
264	>
265	>	if self.debug:
266	>	print "\t results %s \n"%results
267		return results
268
269		def initializeApi(self, protocol ):
270		"""
271		Instantiate storage interface
272		"""
273	+	if self.debug : print 'initializeApi() :\n'
274		self.source_prot = protocol
275		self.dest_prot = protocol
276		if not self.params['source'] : self.source_prot = 'local'
#	Line 164 | Line 279 | class cmscp:
279		Destination_SE = self.storageInterface( self.params['destination'], self.dest_prot )
280
281		if self.debug :
282	<	print '(source=%s,  protocol=%s)'%(self.params['source'], self.source_prot)
283	<	print '(destination=%s,  protocol=%s)'%(self.params['destination'], self.dest_prot)
282	>	msg  = '\t(source=%s,  protocol=%s)'%(self.params['source'], self.source_prot)
283	>	msg += '\t(destination=%s,  protocol=%s)'%(self.params['destination'], self.dest_prot)
284	>	print msg
285
286		return Source_SE, Destination_SE
287
#	Line 173 | Line 289 | class cmscp:
289		"""
290		Make the real file copy using SE API
291		"""
292	+	msg = ""
293		if self.debug :
294	<	print 'copy(): using %s protocol'%protocol
294	>	msg  = 'copy() :\n'
295	>	msg += '\tusing %s protocol\n'%protocol
296	>	print msg
297		try:
298		Source_SE, Destination_SE = self.initializeApi( protocol )
299		except Exception, ex:
300		return self.updateReport('', '-1', str(ex))
301
302		# create remote dir
303	<	if protocol in ['gridftp','rfio']:
303	>	if Destination_SE.protocol in ['gridftp','rfio','srmv2']:
304		try:
305	<	self.createDir( Destination_SE, protocol )
306	<	except Exception, ex:
305	>	self.createDir( Destination_SE, Destination_SE.protocol )
306	>	except OperationException, ex:
307		return self.updateReport('', '60316', str(ex))
308	+	## when the client commands are not found (wrong env or really missing)
309	+	except MissingCommand, ex:
310	+	msg = "ERROR %s %s" %(str(ex), str(ex.detail))
311	+	return self.updateReport('', '10041', msg)
312
313		## prepare for real copy  ##
314		try :
#	Line 193 | Line 316 | class cmscp:
316		sbi_dest = SBinterface(Destination_SE)
317		sbi_source = SBinterface(Source_SE)
318		except ProtocolMismatch, ex:
319	<	msg = str(ex)+'\n'
320	<	msg += "ERROR : Unable to create SBinterface with %s protocol\n"%protocol
321	<	return self.updateReport('', '-1', str(ex))
319	>	msg  = "ERROR : Unable to create SBinterface with %s protocol"%protocol
320	>	msg += str(ex)
321	>	return self.updateReport('', '-1', msg)
322
323		results = {}
324		## loop over the complete list of files
325		for filetocopy in list_file:
326	<	if self.debug : print 'start real copy for %s'%filetocopy
326	>	if self.debug : print '\tStart real copy for %s'%filetocopy
327		try :
328	<	ErCode, msg = self.checkFileExist( sbi_source, sbi_dest, filetocopy )
328	>	ErCode, msg = self.checkFileExist( sbi_source, sbi_dest, filetocopy, options )
329		except Exception, ex:
330		ErCode = -1
331		msg = str(ex)
332		if ErCode == '0':
333		ErCode, msg = self.makeCopy( sbi, filetocopy , options, protocol,sbi_dest )
334	<	if self.debug : print 'Copy results for %s is %s'%( os.path.basename(filetocopy), ErCode)
334	>	if self.debug : print '\tCopy results for %s is %s'%( os.path.basename(filetocopy), ErCode)
335		results.update( self.updateReport(filetocopy, ErCode, msg))
336		return results
337
#	Line 217 | Line 340 | class cmscp:
340		"""
341		Create the storage interface.
342		"""
343	+	if self.debug : print 'storageInterface():\n'
344		try:
345		interface = SElement( FullPath(endpoint), protocol )
346		except ProtocolUnknown, ex:
347	<	msg = ''
348	<	if self.debug : msg = str(ex)+'\n'
225	<	msg += "ERROR : Unable to create interface with %s protocol\n"%protocol
347	>	msg  = "ERROR : Unable to create interface with %s protocol"%protocol
348	>	msg += str(ex)
349		raise Exception(msg)
350
351		return interface
#	Line 232 | Line 355 | class cmscp:
355		Create remote dir for gsiftp REALLY TEMPORARY
356		this should be transparent at SE API level.
357		"""
358	+	if self.debug : print 'createDir():\n'
359		msg = ''
360		try:
361		action = SBinterface( Destination_SE )
362		action.createDir()
363	<	if self.debug: msg+= "The directory has been created using protocol %s\n"%protocol
363	>	if self.debug: print "\tThe directory has been created using protocol %s"%protocol
364		except TransferException, ex:
365	<	msg = str(ex)
365	>	msg  = "ERROR: problem with the directory creation using %s protocol "%protocol
366	>	msg += str(ex)
367		if self.debug :
368	<	msg += str(ex.detail)+'\n'
369	<	msg += str(ex.output)+'\n'
370	<	msg += "ERROR: problem with the directory creation using %s protocol \n"%protocol
371	<	raise Exceptions(msg)
368	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
369	>	dbgmsg += '\t'+str(ex.output)+'\n'
370	>	print dbgmsg
371	>	raise Exception(msg)
372		except OperationException, ex:
373	<	msg = str(ex)
374	<	if self.debug : msg += str(ex.detail)+'\n'
375	<	msg += "ERROR: problem with the directory creation using %s protocol \n"%protocol
376	<
373	>	msg  = "ERROR: problem with the directory creation using %s protocol "%protocol
374	>	msg += str(ex)
375	>	if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n'
376	>	raise Exception(msg)
377	>	except MissingDestination, ex:
378	>	msg  = "ERROR: problem with the directory creation using %s protocol "%protocol
379	>	msg += str(ex)
380	>	if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n'
381	>	raise Exception(msg)
382	>	except AlreadyExistsException, ex:
383	>	if self.debug: print "\tThe directory already exist"
384	>	pass
385		return msg
386
387	<	def checkFileExist( self, sbi_source, sbi_dest, filetocopy ):
387	>	def checkFileExist( self, sbi_source, sbi_dest, filetocopy, option ):
388		"""
389		Check both if source file exist AND
390		if destination file ALREADY exist.
391		"""
392	+	if self.debug : print 'checkFileExist():\n'
393		ErCode = '0'
394		msg = ''
395		f_tocopy=filetocopy
396		if self.source_prot != 'local':f_tocopy = os.path.basename(filetocopy)
397		try:
398	<	checkSource = sbi_source.checkExists( f_tocopy )
398	>	checkSource = sbi_source.checkExists( f_tocopy , opt=option )
399	>	if self.debug : print '\tCheck for local file %s exist succeded \n'%f_tocopy
400		except OperationException, ex:
401	<	msg = str(ex)
401	>	msg  ='ERROR: problems checkig if source file %s exist'%filetocopy
402	>	msg += str(ex)
403		if self.debug :
404	<	msg += str(ex.detail)+'\n'
405	<	msg += str(ex.output)+'\n'
406	<	msg +='ERROR: problems checkig if source file %s exist'%filetocopy
404	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
405	>	dbgmsg += '\t'+str(ex.output)+'\n'
406	>	print dbgmsg
407		raise Exception(msg)
408		except WrongOption, ex:
409	<	msg = str(ex)
409	>	msg  ='ERROR problems checkig if source file % exist'%filetocopy
410	>	msg += str(ex)
411		if self.debug :
412	<	msg += str(ex.detail)+'\n'
413	<	msg += str(ex.output)+'\n'
414	<	msg +='ERROR problems checkig if source file % exist'%filetocopy
412	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
413	>	dbgmsg += '\t'+str(ex.output)+'\n'
414	>	print dbgmsg
415	>	raise Exception(msg)
416	>	except MissingDestination, ex:
417	>	msg  ='ERROR problems checkig if source file % exist'%filetocopy
418	>	msg += str(ex)
419	>	if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n'
420		raise Exception(msg)
421	+	## when the client commands are not found (wrong env or really missing)
422	+	except MissingCommand, ex:
423	+	ErCode = '10041'
424	+	msg = "ERROR %s %s" %(str(ex), str(ex.detail))
425	+	return ErCode, msg
426		if not checkSource :
427		ErCode = '60302'
428		msg = "ERROR file %s do not exist"%os.path.basename(filetocopy)
429		return ErCode, msg
283	–
430		f_tocopy=filetocopy
431		if self.dest_prot != 'local':f_tocopy = os.path.basename(filetocopy)
432		try:
433	<	check = sbi_dest.checkExists( f_tocopy )
433	>	check = sbi_dest.checkExists( f_tocopy, opt=option )
434	>	if self.debug : print '\tCheck for remote file %s exist succeded \n'%f_tocopy
435		except OperationException, ex:
436	<	msg = str(ex)
436	>	msg  = 'ERROR: problems checkig if file %s already exist'%filetocopy
437	>	msg += str(ex)
438		if self.debug :
439	<	msg += str(ex.detail)+'\n'
440	<	msg += str(ex.output)+'\n'
441	<	msg +='ERROR: problems checkig if file %s already exist'%filetocopy
439	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
440	>	dbgmsg += '\t'+str(ex.output)+'\n'
441	>	print dbgmsg
442		raise Exception(msg)
443		except WrongOption, ex:
444	<	msg = str(ex)
444	>	msg  = 'ERROR problems checkig if file % already exist'%filetocopy
445	>	msg += str(ex)
446		if self.debug :
447	<	msg += str(ex.detail)+'\n'
448	<	msg += str(ex.output)+'\n'
449	<	msg +='ERROR problems checkig if file % already exist'%filetocopy
447	>	msg += '\t'+msg+'\n\t'+str(ex.detail)+'\n'
448	>	msg += '\t'+str(ex.output)+'\n'
449	>	raise Exception(msg)
450	>	except MissingDestination, ex:
451	>	msg  ='ERROR problems checkig if source file % exist'%filetocopy
452	>	msg += str(ex)
453	>	if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n'
454		raise Exception(msg)
455	+	## when the client commands are not found (wrong env or really missing)
456	+	except MissingCommand, ex:
457	+	ErCode = '10041'
458	+	msg = "ERROR %s %s" %(str(ex), str(ex.detail))
459	+	return ErCode, msg
460		if check :
461		ErCode = '60303'
462		msg = "file %s already exist"%os.path.basename(filetocopy)
#	Line 309 | Line 467 | class cmscp:
467		"""
468		call the copy API.
469		"""
470	+	if self.debug : print 'makeCopy():\n'
471		path = os.path.dirname(filetocopy)
472		file_name =  os.path.basename(filetocopy)
473		source_file = filetocopy
#	Line 316 | Line 475 | class cmscp:
475		if self.params['source'] == '' and path == '':
476		source_file = os.path.abspath(filetocopy)
477		elif self.params['destination'] =='':
478	<	dest_file = os.path.join(os.getcwd(),file_name)
478	>	destDir = self.params.get('destinationDir',os.getcwd())
479	>	dest_file = os.path.join(destDir,file_name)
480		elif self.params['source'] != '' and self.params['destination'] != '' :
481		source_file = file_name
482
#	Line 326 | Line 486 | class cmscp:
486		try:
487		sbi.copy( source_file , dest_file , opt = option)
488		except TransferException, ex:
489	<	msg = str(ex)
489	>	msg  = "Problem copying %s file" % filetocopy
490	>	msg += str(ex)
491		if self.debug :
492	<	msg += str(ex.detail)+'\n'
493	<	msg += str(ex.output)+'\n'
494	<	msg += "Problem copying %s file" % filetocopy
492	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
493	>	dbgmsg += '\t'+str(ex.output)+'\n'
494	>	print dbgmsg
495		ErCode = '60307'
496		except WrongOption, ex:
497	<	msg = str(ex)
497	>	msg  = "Problem copying %s file" % filetocopy
498	>	msg += str(ex)
499	>	if self.debug :
500	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
501	>	dbgmsg += '\t'+str(ex.output)+'\n'
502	>	print dbgmsg
503	>	except SizeZeroException, ex:
504	>	msg  = "Problem copying %s file" % filetocopy
505	>	msg += str(ex)
506	>	if self.debug :
507	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
508	>	dbgmsg += '\t'+str(ex.output)+'\n'
509	>	print dbgmsg
510	>	ErCode = '60307'
511	>	## when the client commands are not found (wrong env or really missing)
512	>	except MissingCommand, ex:
513	>	ErCode = '10041'
514	>	msg  = "Problem copying %s file" % filetocopy
515	>	msg += str(ex)
516		if self.debug :
517	<	msg += str(ex.detail)+'\n'
518	<	msg += str(ex.output)+'\n'
519	<	msg += "Problem copying %s file" % filetocopy
517	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
518	>	dbgmsg += '\t'+str(ex.output)+'\n'
519	>	print dbgmsg
520	>	except AuthorizationException, ex:
521		ErCode = '60307'
522	<	if ErCode == '0' and protocol.find('srm') == 0:
522	>	msg  = "Problem copying %s file" % filetocopy
523	>	msg += str(ex)
524	>	if self.debug :
525	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
526	>	dbgmsg += '\t'+str(ex.output)+'\n'
527	>	print dbgmsg
528	>	if ErCode == '0' and protocol.find('srmv') == 0:
529		remote_file_size = -1
530		local_file_size = os.path.getsize( source_file )
531		try:
532	<	remote_file_size = sbi_dest.getSize( dest_file )
532	>	remote_file_size = sbi_dest.getSize( dest_file, opt=option )
533	>	if self.debug : print '\t Check of remote size succeded for file %s\n'%dest_file
534		except TransferException, ex:
535	<	msg = str(ex)
535	>	msg  = "Problem checking the size of %s file" % filetocopy
536	>	msg += str(ex)
537		if self.debug :
538	<	msg += str(ex.detail)+'\n'
539	<	msg += str(ex.output)+'\n'
540	<	msg += "Problem checking the size of %s file" % filetocopy
538	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
539	>	dbgmsg += '\t'+str(ex.output)+'\n'
540	>	print dbgmsg
541		ErCode = '60307'
542		except WrongOption, ex:
543	<	msg = str(ex)
543	>	msg  = "Problem checking the size of %s file" % filetocopy
544	>	msg += str(ex)
545		if self.debug :
546	<	msg += str(ex.detail)+'\n'
547	<	msg += str(ex.output)+'\n'
548	<	msg += "Problem checking the size of %s file" % filetocopy
546	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
547	>	dbgmsg += '\t'+str(ex.output)+'\n'
548	>	print dbgmsg
549		ErCode = '60307'
550		if local_file_size != remote_file_size:
551		msg = "File size dosn't match: local size = %s ; remote size = %s " % (local_file_size, remote_file_size)
552		ErCode = '60307'
553
554	+	if ErCode != '0':
555	+	try :
556	+	self.removeFile( sbi_dest, dest_file, option )
557	+	except Exception, ex:
558	+	msg += '\n'+str(ex)
559		return ErCode, msg
560
561	<	def backup(self):
562	<	"""
563	<	Check infos from TFC using existing api obtaining:
564	<	1)destination
565	<	2)protocol
566	<	"""
567	<	return
561	>	def removeFile( self, sbi_dest, filetocopy, option ):
562	>	"""
563	>	"""
564	>	if self.debug : print 'removeFile():\n'
565	>	f_tocopy=filetocopy
566	>	if self.dest_prot != 'local':f_tocopy = os.path.basename(filetocopy)
567	>	try:
568	>	sbi_dest.delete( f_tocopy, opt=option )
569	>	if self.debug : '\t deletion of file %s succeeded\n'%str(filetocopy)
570	>	except OperationException, ex:
571	>	msg  ='ERROR: problems removing partially staged file %s'%filetocopy
572	>	msg += str(ex)
573	>	if self.debug :
574	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
575	>	dbgmsg += '\t'+str(ex.output)+'\n'
576	>	print dbgmsg
577	>	raise Exception(msg)
578	>
579	>	return
580
581		def updateReport(self, file, erCode, reason, lfn='', se='' ):
582		"""
#	Line 394 | Line 600 | class cmscp:
600		cmscp_exit_status = 0
601		txt = ''
602		for file, dict in results.iteritems():
603	+	reason = str(dict['reason'])
604	+	if str(reason).find("'") > -1:
605	+	reason = " ".join(reason.split("'"))
606	+	reason="'%s'"%reason
607		if file:
608		if dict['lfn']=='':
609		lfn = '$LFNBaseName/'+os.path.basename(file)
#	Line 402 | Line 612 | class cmscp:
612		lfn = dict['lfn']+os.path.basename(file)
613		se = dict['se']
614		#dict['lfn'] # to be implemented
615	<	txt +=  'echo "Report for File: '+file+'"\n'
616	<	txt +=  'echo "LFN: '+lfn+'"\n'
617	<	txt +=  'echo "StorageElement: '+se+'"\n'
618	<	txt += 'echo "StageOutExitStatusReason ='+dict['reason']+'" | tee -a $RUNTIME_AREA/$repo\n'
615	>	txt += 'echo "Report for File: '+file+'"\n'
616	>	txt += 'echo "LFN: '+lfn+'"\n'
617	>	txt += 'echo "StorageElement: '+se+'"\n'
618	>	txt += 'echo "StageOutExitStatusReason = %s" | tee -a $RUNTIME_AREA/$repo\n'%reason
619		txt += 'echo "StageOutSE = '+se+'" >> $RUNTIME_AREA/$repo\n'
620	+	#txt += 'export LFNBaseName='+lfn+'\n'
621	+	txt += 'export SE='+se+'\n'
622	+
623		if dict['erCode'] != '0':
624		cmscp_exit_status = dict['erCode']
412	–	cmscp_exit_status = dict['erCode']
625		else:
626	+	txt += 'echo "StageOutExitStatusReason = %s" | tee -a $RUNTIME_AREA/$repo\n'%reason
627		cmscp_exit_status = dict['erCode']
628		cmscp_exit_status = dict['erCode']
629		txt += '\n'
#	Line 424 | Line 637 | class cmscp:
637		def usage():
638
639		msg="""
640	<	required parameters:
641	<	--source        :: REMOTE           :
642	<	--destination   :: REMOTE           :
430	<	--debug             :
431	<	--inFile :: absPath : or name NOT RELATIVE PATH
432	<	--outFIle :: onlyNAME : NOT YET SUPPORTED
640	>	cmscp:
641	>	safe copy of local file  to/from remote SE via lcg_cp/srmcp,
642	>	including success checking  version also for CAF using rfcp command to copy the output to SE
643
644	<	optional parameters
644	>	accepted parameters:
645	>	source           =
646	>	destination      =
647	>	inputFileList    =
648	>	outputFileList   =
649	>	protocol         =
650	>	option           =
651	>	middleware       =
652	>	srm_version      =
653	>	destinationDir   =
654	>	lfn=             =
655	>	local_stage      =  activate stage fall back
656	>	debug            =  activate verbose print out
657	>	help             =  print on line man and exit
658	>
659	>	mandatory:
660	>	* "source" and/or "destination" must always be defined
661	>	* either "middleware" or "protocol" must always be defined
662	>	* "inputFileList" must always be defined
663	>	* if "local_stage" = 1 also  "lfn" must be defined
664		"""
665		print msg
666
#	Line 459 | Line 688 | if __name__ == '__main__' :
688		import getopt
689
690		allowedOpt = ["source=", "destination=", "inputFileList=", "outputFileList=", \
691	<	"protocol=","option=", "middleware=", "srm_version=", "debug", "help"]
691	>	"protocol=","option=", "middleware=", "srm_version=", \
692	>	"destinationDir=", "lfn=", "local_stage", "debug", "help"]
693		try:
694		opts, args = getopt.getopt( sys.argv[1:], "", allowedOpt )
695		except getopt.GetoptError, err:

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines