ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/cmscp.py

Comparing COMP/CRAB/python/cmscp.py (file contents):
Revision 1.20 by spiga, Tue Oct 28 17:40:55 2008 UTC vs.
Revision 1.67 by fanzago, Mon Jan 25 11:15:01 2010 UTC

#	Line 1 | Line 1
1		#!/usr/bin/env python
2	–
2		import sys, os
3		from ProdCommon.Storage.SEAPI.SElement import SElement, FullPath
4		from ProdCommon.Storage.SEAPI.SBinterface import *
#	Line 10 | Line 9 | class cmscp:
9		def __init__(self, args):
10		"""
11		cmscp
12	<
14	<	safe copy of local file in current directory to remote SE via lcg_cp/srmcp,
12	>	safe copy of local file  to/from remote SE via lcg_cp/srmcp,
13		including success checking  version also for CAF using rfcp command to copy the output to SE
14		input:
15		$1 middleware (CAF, LSF, LCG, OSG)
#	Line 19 | Line 17 | class cmscp:
17		$3 if needed: file name (the output file name)
18		$5 remote SE (complete endpoint)
19		$6 srm version
20	+	--lfn $LFNBaseName
21		output:
22		return 0 if all ok
23		return 60307 if srmcp failed
#	Line 26 | Line 25 | class cmscp:
25		"""
26
27		#set default
28	<	self.params = {"source":'', "destination":'', "inputFileList":'', "outputFileList":'', \
29	<	"protocol":'', "option":'', "middleware":'', "srm_version":'srmv2'}
28	>	self.params = {"source":'', "destination":'','destinationDir':'', "inputFileList":'', "outputFileList":'', \
29	>	"protocol":'', "option":'', "middleware":'', "srm_version":'srmv2', "lfn":'' }
30		self.debug = 0
31	<
31	>	self.local_stage = 0
32		self.params.update( args )
33
34		return
#	Line 38 | Line 37 | class cmscp:
37		"""
38		check command line parameter
39		"""
41	–
40		if 'help' in self.params.keys(): HelpOptions()
41		if 'debug' in self.params.keys(): self.debug = 1
42	+	if 'local_stage' in self.params.keys(): self.local_stage = 1
43
44		# source and dest cannot be undefined at same time
45		if not self.params['source']  and not self.params['destination'] :
46		HelpOptions()
48	–
47		# if middleware is not defined --> protocol cannot be empty
48		if not self.params['middleware'] and not self.params['protocol'] :
49		HelpOptions()
50
51		# input file must be defined
52	<	if not self.params['inputFileList'] : HelpOptions()
52	>	if not self.params['inputFileList'] :
53	>	HelpOptions()
54		else:
55		file_to_copy=[]
56		if self.params['inputFileList'].find(','):
#	Line 60 | Line 59 | class cmscp:
59		file_to_copy.append(self.params['inputFileList'])
60		self.params['inputFileList'] = file_to_copy
61
62	+	if not self.params['lfn'] and self.local_stage == 1 : HelpOptions()
63	+
64		## TO DO:
65		#### add check for outFiles
66		#### add map {'inFileNAME':'outFileNAME'} to change out name
67
67	–	return
68
69		def run( self ):
70		"""
71		Check if running on UI (no $middleware) or
72		on WN (on the Grid), and take different action
73		"""
74	–
74		self.processOptions()
75	+	if self.debug: print 'calling run() : \n'
76		# stage out from WN
77		if self.params['middleware'] :
78	<	results = self.stager(self.params['middleware'],self.params['inputFileList'])
79	<	self.finalReport(results)
78	>	results = self.stager(self.params['middleware'],self.params['inputFileList'])
79	>	self.finalReport(results)
80		# Local interaction with SE
81		else:
82	<	results = self.copy(self.params['inputFilesList'], self.params['protocol'], self.params['option'] )
83	<	return results
82	>	results = self.copy(self.params['inputFileList'], self.params['protocol'], self.params['option'] )
83	>	return results
84	>
85	>	def checkLcgUtils( self ):
86	>	"""
87	>	_checkLcgUtils_
88	>	check the lcg-utils version and report
89	>	"""
90	>	import commands
91	>	cmd = "lcg-cp --version | grep lcg_util"
92	>	status, output = commands.getstatusoutput( cmd )
93	>	num_ver = -1
94	>	if output.find("not found") == -1 or status == 0:
95	>	temp = output.split("-")
96	>	version = ""
97	>	if len(temp) >= 2:
98	>	version = output.split("-")[1]
99	>	temp = version.split(".")
100	>	if len(temp) >= 1:
101	>	num_ver = int(temp[0])*10
102	>	num_ver += int(temp[1])
103	>	return num_ver
104
105		def setProtocol( self, middleware ):
106		"""
#	Line 88 | Line 108 | class cmscp:
108		which depend on scheduler
109		"""
110		# default To be used with "middleware"
111	+	if self.debug:
112	+	print 'setProtocol() :\n'
113	+	print '\tmiddleware =  %s utils \n'%middleware
114	+
115		lcgOpt={'srmv1':'-b -D srmv1  -t 2400 --verbose',
116		'srmv2':'-b -D srmv2  -t 2400 --verbose'}
117	+	if self.checkLcgUtils() >= 17:
118	+	lcgOpt={'srmv1':'-b -D srmv1 --srm-timeout 2400 --sendreceive-timeout 2400 --connect-timeout 2400 --verbose',
119	+	'srmv2':'-b -D srmv2 --srm-timeout 2400 --sendreceive-timeout 2400 --connect-timeout 2400 --verbose'}
120	+
121		srmOpt={'srmv1':' -report ./srmcp.report -retry_timeout 480000 -retry_num 3 -streams_num=1 ',
122	<	'srmv2':' -report ./srmcp.report -retry_timeout 480000 -retry_num 3 '}
122	>	'srmv2':' -report=./srmcp.report -retry_timeout=480000 -retry_num=3 '}
123		rfioOpt=''
124
125		supported_protocol = None
126	<	if middleware.lower() in ['osg','lcg','condor']:
126	>	if middleware.lower() in ['osg','lcg','condor','sge']:
127		supported_protocol = [('srm-lcg',lcgOpt[self.params['srm_version']]),\
128	<	(self.params['srm_version'],srmOpt[self.params['srm_version']])]
128	>	(self.params['srm_version'],srmOpt[self.params['srm_version']])]
129		elif middleware.lower() in ['lsf','caf']:
130		supported_protocol = [('rfio',rfioOpt)]
131	+	elif middleware.lower() in ['pbs']:
132	+	supported_protocol = [('rfio',rfioOpt),('local','')]
133	+	elif middleware.lower() in ['arc']:
134	+	supported_protocol = [('srmv2','-debug'),('srmv1','-debug')]
135		else:
136		## here we can add support for any kind of protocol,
137		## maybe some local schedulers need something dedicated
138		pass
139		return supported_protocol
140
141	+
142	+	def checkCopy (self, copy_results, len_list_files, prot, lfn='', se=''):
143	+	"""
144	+	Checks the status of copy and update result dictionary
145	+	"""
146	+	list_retry = []
147	+	list_retry_localSE = []
148	+	list_not_existing = []
149	+	list_ok = []
150	+
151	+	if self.debug:
152	+	print 'in checkCopy() :\n'
153	+	for file, dict in copy_results.iteritems():
154	+	er_code = dict['erCode']
155	+	if er_code == '0':
156	+	list_ok.append(file)
157	+	reason = 'Copy succedeed with %s utils'%prot
158	+	dict['reason'] = reason
159	+	elif er_code == '60302':
160	+	list_not_existing.append( file )
161	+	elif er_code == '10041':
162	+	list_retry.append( file )
163	+	## WHAT TO DO IN GENERAL FAILURE CONDITION
164	+	else:
165	+	list_retry_localSE.append( file )
166	+
167	+	if self.debug:
168	+	print "\t file %s \n"%file
169	+	print "\t dict['erCode'] %s \n"%dict['erCode']
170	+	print "\t dict['reason'] %s \n"%dict['reason']
171	+
172	+	if (lfn != '') and (se != ''):
173	+	upDict = self.updateReport(file, er_code, dict['reason'], lfn, se)
174	+	else:
175	+	upDict = self.updateReport(file, er_code, dict['reason'])
176	+
177	+	copy_results.update(upDict)
178	+
179	+	msg = ''
180	+	if len(list_ok) != 0:
181	+	msg += '\tCopy of %s succedeed with %s utils\n'%(str(list_ok),prot)
182	+	if len(list_ok) != len_list_files :
183	+	msg += '\tCopy of %s failed using %s for files \n'%(str(list_retry),prot)
184	+	msg += '\tCopy of %s failed using %s : files not found \n'%(str(list_not_existing),prot)
185	+	if self.debug : print msg
186	+
187	+	return copy_results, list_ok, list_retry, list_retry_localSE
188	+
189	+	def LocalCopy(self, list_retry, results):
190	+	"""
191	+	Tries the stage out to the CloseSE
192	+	"""
193	+	if self.debug:
194	+	print 'in LocalCopy() :\n'
195	+	print '\t list_retry %s utils \n'%list_retry
196	+	print '\t len(list_retry) %s \n'%len(list_retry)
197	+
198	+	list_files = list_retry
199	+	self.params['inputFilesList']=list_files
200	+
201	+	### copy backup
202	+	from ProdCommon.FwkJobRep.SiteLocalConfig import loadSiteLocalConfig
203	+	siteCfg = loadSiteLocalConfig()
204	+	seName = siteCfg.localStageOut.get("se-name", None)
205	+	catalog = siteCfg.localStageOut.get("catalog", None)
206	+	implName = siteCfg.localStageOut.get("command", None)
207	+	if (implName == 'srm'):
208	+	implName='srmv1'
209	+	self.params['srm_version']=implName
210	+	##### to be improved ###############
211	+	if (implName == 'rfcp'):
212	+	self.params['middleware']='lsf'
213	+	####################################
214	+
215	+	self.params['protocol']=implName
216	+	tfc = siteCfg.trivialFileCatalog()
217	+
218	+	if self.debug:
219	+	print '\t siteCFG %s \n'%siteCfg
220	+	print '\t seName %s \n'%seName
221	+	print '\t catalog %s \n'%catalog
222	+	print "\t self.params['protocol'] %s \n"%self.params['protocol']
223	+	print '\t tfc %s '%tfc
224	+	print "\t self.params['inputFilesList'] %s \n"%self.params['inputFilesList']
225	+
226	+	if (str(self.params['lfn']).find("/store/") != -1):
227	+	temp = str(self.params['lfn']).split("/store/")
228	+	self.params['lfn']= "/store/temp/" + temp[1]
229	+
230	+	file_backup=[]
231	+	for input in self.params['inputFilesList']:
232	+	file = self.params['lfn'] + os.path.basename(input)
233	+	surl = tfc.matchLFN(tfc.preferredProtocol, file)
234	+	file_backup.append(surl)
235	+	if self.debug:
236	+	print '\t lfn %s \n'%self.params['lfn']
237	+	print '\t file %s \n'%file
238	+	print '\t surl %s \n'%surl
239	+
240	+	destination=os.path.dirname(file_backup[0])
241	+	if ( destination[-1] != '/' ) : destination = destination + '/'
242	+	self.params['destination']=destination
243	+
244	+	if self.debug:
245	+	print "\t self.params['destination']%s \n"%self.params['destination']
246	+	print "\t self.params['protocol'] %s \n"%self.params['protocol']
247	+	print "\t self.params['option']%s \n"%self.params['option']
248	+
249	+	for prot, opt in self.setProtocol( self.params['middleware'] ):
250	+	if self.debug: print '\tIn LocalCopy trying the stage out with %s utils \n'%prot
251	+	localCopy_results = self.copy( self.params['inputFileList'], prot, opt, backup='yes' )
252	+	if localCopy_results.keys() == [''] or localCopy_results.keys() == '' :
253	+	results.update(localCopy_results)
254	+	else:
255	+	localCopy_results, list_ok, list_retry, list_retry_localSE = self.checkCopy(localCopy_results, len(list_files), prot, self.params['lfn'], seName)
256	+	results.update(localCopy_results)
257	+	if len(list_ok) == len(list_files) :
258	+	break
259	+	if len(list_retry):
260	+	list_files = list_retry
261	+	else: break
262	+	if self.debug:
263	+	print "\t localCopy_results = %s \n"%localCopy_results
264	+
265	+	return results
266	+
267		def stager( self, middleware, list_files ):
268		"""
269		Implement the logic for remote stage out
270		"""
271	+
272	+	if self.debug:
273	+	print 'stager() :\n'
274	+	print '\tmiddleware %s\n'%middleware
275	+	print '\tlist_files %s\n'%list_files
276	+
277		results={}
278		for prot, opt in self.setProtocol( middleware ):
279	<	if self.debug: print 'Trying stage out with %s utils \n'%prot
279	>	if self.debug: print '\tTrying the stage out with %s utils \n'%prot
280		copy_results = self.copy( list_files, prot, opt )
281	<	list_retry = []
118	<	list_existing = []
119	<	list_ok = []
120	<	if copy_results.keys() == '':
281	>	if copy_results.keys() == [''] or copy_results.keys() == '' :
282		results.update(copy_results)
283		else:
284	<	for file, dict in copy_results.iteritems():
124	<	er_code = dict['erCode']
125	<	if er_code == '0':
126	<	list_ok.append(file)
127	<	reason = 'Copy succedeed with %s utils'%prot
128	<	upDict = self.updateReport(file, er_code, reason)
129	<	copy_results.update(upDict)
130	<	elif er_code == '60303': list_existing.append( file )
131	<	else: list_retry.append( file )
284	>	copy_results, list_ok, list_retry, list_retry_localSE = self.checkCopy(copy_results, len(list_files), prot)
285		results.update(copy_results)
133	–	if len(list_ok) != 0:
134	–	msg = 'Copy of %s succedeed with %s utils\n'%(str(list_ok),prot)
135	–	if self.debug : print msg
286		if len(list_ok) == len(list_files) :
287		break
288	<	else:
289	<	if self.debug : print 'Copy of files %s failed using %s...\n'%(str(list_retry)+str(list_existing),prot)
290	<	if len(list_retry): list_files = list_retry
291	<	else: break
292	<
293	<	#### TODO Daniele
294	<	#check is something fails and created related dict
295	<	#      backup = self.analyzeResults(results)
296	<
297	<	#      if backup :
148	<	#          msg = 'WARNING: backup logic is under implementation\n'
149	<	#          #backupDict = self.backup()
150	<	#          ### NOTE: IT MUST RETURN a DICT contains also LFN and SE Name
151	<	#          results.update(backupDict)
152	<	#          print msg
288	>	if len(list_retry):
289	>	list_files = list_retry
290	>	else: break
291	>
292	>	if self.local_stage:
293	>	if len(list_retry_localSE):
294	>	results = self.LocalCopy(list_retry_localSE, results)
295	>
296	>	if self.debug:
297	>	print "\t results %s \n"%results
298		return results
299
300		def initializeApi(self, protocol ):
301		"""
302		Instantiate storage interface
303		"""
304	+	if self.debug : print 'initializeApi() :\n'
305		self.source_prot = protocol
306		self.dest_prot = protocol
307		if not self.params['source'] : self.source_prot = 'local'
#	Line 164 | Line 310 | class cmscp:
310		Destination_SE = self.storageInterface( self.params['destination'], self.dest_prot )
311
312		if self.debug :
313	<	print '(source=%s,  protocol=%s)'%(self.params['source'], self.source_prot)
314	<	print '(destination=%s,  protocol=%s)'%(self.params['destination'], self.dest_prot)
313	>	msg  = '\t(source=%s,  protocol=%s)'%(self.params['source'], self.source_prot)
314	>	msg += '\t(destination=%s,  protocol=%s)'%(self.params['destination'], self.dest_prot)
315	>	print msg
316
317		return Source_SE, Destination_SE
318
319	<	def copy( self, list_file, protocol, options ):
319	>	def copy( self, list_file, protocol, options, backup='no' ):
320		"""
321		Make the real file copy using SE API
322		"""
323	+	msg = ""
324		if self.debug :
325	<	print 'copy(): using %s protocol'%protocol
325	>	msg  = 'copy() :\n'
326	>	msg += '\tusing %s protocol\n'%protocol
327	>	print msg
328		try:
329		Source_SE, Destination_SE = self.initializeApi( protocol )
330		except Exception, ex:
331		return self.updateReport('', '-1', str(ex))
332
333		# create remote dir
334	<	if protocol in ['gridftp','rfio']:
334	>	if Destination_SE.protocol in ['gridftp','rfio','srmv2']:
335		try:
336	<	self.createDir( Destination_SE, protocol )
337	<	except Exception, ex:
336	>	self.createDir( Destination_SE, Destination_SE.protocol )
337	>	except OperationException, ex:
338		return self.updateReport('', '60316', str(ex))
339	+	## when the client commands are not found (wrong env or really missing)
340	+	except MissingCommand, ex:
341	+	msg = "ERROR %s %s" %(str(ex), str(ex.detail))
342	+	return self.updateReport('', '10041', msg)
343
344		## prepare for real copy  ##
345		try :
#	Line 193 | Line 347 | class cmscp:
347		sbi_dest = SBinterface(Destination_SE)
348		sbi_source = SBinterface(Source_SE)
349		except ProtocolMismatch, ex:
350	<	msg = str(ex)+'\n'
351	<	msg += "ERROR : Unable to create SBinterface with %s protocol\n"%protocol
352	<	return self.updateReport('', '-1', str(ex))
350	>	msg  = "ERROR : Unable to create SBinterface with %s protocol"%protocol
351	>	msg += str(ex)
352	>	return self.updateReport('', '-1', msg)
353
354		results = {}
355		## loop over the complete list of files
356		for filetocopy in list_file:
357	<	if self.debug : print 'start real copy for %s'%filetocopy
357	>	if self.debug : print '\tStart real copy for %s'%filetocopy
358		try :
359	<	ErCode, msg = self.checkFileExist( sbi_source, sbi_dest, filetocopy )
359	>	ErCode, msg = self.checkFileExist( sbi_source, sbi_dest, filetocopy, options )
360		except Exception, ex:
361	<	ErCode = -1
361	>	ErCode = '60307'
362		msg = str(ex)
363		if ErCode == '0':
364		ErCode, msg = self.makeCopy( sbi, filetocopy , options, protocol,sbi_dest )
365	<	if self.debug : print 'Copy results for %s is %s'%( os.path.basename(filetocopy), ErCode)
365	>	if (ErCode == '0') and (backup == 'yes'):
366	>	ErCode = '60308'
367	>	if self.debug : print '\tCopy results for %s is %s'%( os.path.basename(filetocopy), ErCode)
368		results.update( self.updateReport(filetocopy, ErCode, msg))
369		return results
370
#	Line 217 | Line 373 | class cmscp:
373		"""
374		Create the storage interface.
375		"""
376	+	if self.debug : print 'storageInterface():\n'
377		try:
378		interface = SElement( FullPath(endpoint), protocol )
379		except ProtocolUnknown, ex:
380	<	msg = ''
381	<	if self.debug : msg = str(ex)+'\n'
225	<	msg += "ERROR : Unable to create interface with %s protocol\n"%protocol
380	>	msg  = "ERROR : Unable to create interface with %s protocol"%protocol
381	>	msg += str(ex)
382		raise Exception(msg)
383
384		return interface
#	Line 232 | Line 388 | class cmscp:
388		Create remote dir for gsiftp REALLY TEMPORARY
389		this should be transparent at SE API level.
390		"""
391	+	if self.debug : print 'createDir():\n'
392		msg = ''
393		try:
394		action = SBinterface( Destination_SE )
395		action.createDir()
396	<	if self.debug: msg+= "The directory has been created using protocol %s\n"%protocol
396	>	if self.debug: print "\tThe directory has been created using protocol %s"%protocol
397		except TransferException, ex:
398	<	msg = str(ex)
398	>	msg  = "ERROR: problem with the directory creation using %s protocol "%protocol
399	>	msg += str(ex)
400		if self.debug :
401	<	msg += str(ex.detail)+'\n'
402	<	msg += str(ex.output)+'\n'
403	<	msg += "ERROR: problem with the directory creation using %s protocol \n"%protocol
404	<	raise Exceptions(msg)
401	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
402	>	dbgmsg += '\t'+str(ex.output)+'\n'
403	>	print dbgmsg
404	>	raise Exception(msg)
405		except OperationException, ex:
406	<	msg = str(ex)
407	<	if self.debug : msg += str(ex.detail)+'\n'
408	<	msg += "ERROR: problem with the directory creation using %s protocol \n"%protocol
409	<
406	>	msg  = "ERROR: problem with the directory creation using %s protocol "%protocol
407	>	msg += str(ex)
408	>	if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n'
409	>	raise Exception(msg)
410	>	except MissingDestination, ex:
411	>	msg  = "ERROR: problem with the directory creation using %s protocol "%protocol
412	>	msg += str(ex)
413	>	if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n'
414	>	raise Exception(msg)
415	>	except AlreadyExistsException, ex:
416	>	if self.debug: print "\tThe directory already exist"
417	>	pass
418		return msg
419
420	<	def checkFileExist( self, sbi_source, sbi_dest, filetocopy ):
420	>	def checkFileExist( self, sbi_source, sbi_dest, filetocopy, option ):
421		"""
422		Check both if source file exist AND
423		if destination file ALREADY exist.
424		"""
425	+	if self.debug : print 'checkFileExist():\n'
426		ErCode = '0'
427		msg = ''
428		f_tocopy=filetocopy
429		if self.source_prot != 'local':f_tocopy = os.path.basename(filetocopy)
430		try:
431	<	checkSource = sbi_source.checkExists( f_tocopy )
431	>	checkSource = sbi_source.checkExists( f_tocopy , opt=option )
432	>	if self.debug : print '\tCheck for local file %s exist succeded \n'%f_tocopy
433		except OperationException, ex:
434	<	msg = str(ex)
434	>	msg  ='ERROR: problems checkig if source file %s exist'%filetocopy
435	>	msg += str(ex)
436		if self.debug :
437	<	msg += str(ex.detail)+'\n'
438	<	msg += str(ex.output)+'\n'
439	<	msg +='ERROR: problems checkig if source file %s exist'%filetocopy
437	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
438	>	dbgmsg += '\t'+str(ex.output)+'\n'
439	>	print dbgmsg
440		raise Exception(msg)
441		except WrongOption, ex:
442	<	msg = str(ex)
442	>	msg  ='ERROR problems checkig if source file % exist'%filetocopy
443	>	msg += str(ex)
444		if self.debug :
445	<	msg += str(ex.detail)+'\n'
446	<	msg += str(ex.output)+'\n'
447	<	msg +='ERROR problems checkig if source file % exist'%filetocopy
445	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
446	>	dbgmsg += '\t'+str(ex.output)+'\n'
447	>	print dbgmsg
448	>	raise Exception(msg)
449	>	except MissingDestination, ex:
450	>	msg  ='ERROR problems checkig if source file % exist'%filetocopy
451	>	msg += str(ex)
452	>	if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n'
453		raise Exception(msg)
454	+	## when the client commands are not found (wrong env or really missing)
455	+	except MissingCommand, ex:
456	+	ErCode = '10041'
457	+	msg = "ERROR %s %s" %(str(ex), str(ex.detail))
458	+	return ErCode, msg
459		if not checkSource :
460		ErCode = '60302'
461		msg = "ERROR file %s do not exist"%os.path.basename(filetocopy)
462		return ErCode, msg
283	–
463		f_tocopy=filetocopy
464		if self.dest_prot != 'local':f_tocopy = os.path.basename(filetocopy)
465		try:
466	<	check = sbi_dest.checkExists( f_tocopy )
466	>	check = sbi_dest.checkExists( f_tocopy, opt=option )
467	>	if self.debug : print '\tCheck for remote file %s exist succeded \n'%f_tocopy
468		except OperationException, ex:
469	<	msg = str(ex)
469	>	msg  = 'ERROR: problems checkig if file %s already exist'%filetocopy
470	>	msg += str(ex)
471		if self.debug :
472	<	msg += str(ex.detail)+'\n'
473	<	msg += str(ex.output)+'\n'
474	<	msg +='ERROR: problems checkig if file %s already exist'%filetocopy
472	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
473	>	dbgmsg += '\t'+str(ex.output)+'\n'
474	>	print dbgmsg
475		raise Exception(msg)
476		except WrongOption, ex:
477	<	msg = str(ex)
477	>	msg  = 'ERROR problems checkig if file % already exist'%filetocopy
478	>	msg += str(ex)
479		if self.debug :
480	<	msg += str(ex.detail)+'\n'
481	<	msg += str(ex.output)+'\n'
482	<	msg +='ERROR problems checkig if file % already exist'%filetocopy
480	>	msg += '\t'+msg+'\n\t'+str(ex.detail)+'\n'
481	>	msg += '\t'+str(ex.output)+'\n'
482	>	raise Exception(msg)
483	>	except MissingDestination, ex:
484	>	msg  ='ERROR problems checkig if source file % exist'%filetocopy
485	>	msg += str(ex)
486	>	if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n'
487		raise Exception(msg)
488	+	## when the client commands are not found (wrong env or really missing)
489	+	except MissingCommand, ex:
490	+	ErCode = '10041'
491	+	msg = "ERROR %s %s" %(str(ex), str(ex.detail))
492	+	return ErCode, msg
493		if check :
494		ErCode = '60303'
495		msg = "file %s already exist"%os.path.basename(filetocopy)
#	Line 309 | Line 500 | class cmscp:
500		"""
501		call the copy API.
502		"""
503	+	if self.debug : print 'makeCopy():\n'
504		path = os.path.dirname(filetocopy)
505		file_name =  os.path.basename(filetocopy)
506		source_file = filetocopy
#	Line 316 | Line 508 | class cmscp:
508		if self.params['source'] == '' and path == '':
509		source_file = os.path.abspath(filetocopy)
510		elif self.params['destination'] =='':
511	<	dest_file = os.path.join(os.getcwd(),file_name)
511	>	destDir = self.params.get('destinationDir',os.getcwd())
512	>	dest_file = os.path.join(destDir,file_name)
513		elif self.params['source'] != '' and self.params['destination'] != '' :
514		source_file = file_name
515
516		ErCode = '0'
517		msg = ''
518
519	+	if  self.params['option'].find('space_token')>0:
520	+	space_token=self.params['option'].split('=')[1]
521	+	if protocol == 'srmv2': option = '%s -space_token=%s'%(option,space_token)
522	+	if protocol == 'srm-lcg': option = '%s -S %s'%(option,space_token)
523		try:
524		sbi.copy( source_file , dest_file , opt = option)
525		except TransferException, ex:
526	<	msg = str(ex)
526	>	msg  = "Problem copying %s file" % filetocopy
527	>	msg += str(ex)
528		if self.debug :
529	<	msg += str(ex.detail)+'\n'
530	<	msg += str(ex.output)+'\n'
531	<	msg += "Problem copying %s file" % filetocopy
529	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
530	>	dbgmsg += '\t'+str(ex.output)+'\n'
531	>	print dbgmsg
532		ErCode = '60307'
533		except WrongOption, ex:
534	<	msg = str(ex)
534	>	msg  = "Problem copying %s file" % filetocopy
535	>	msg += str(ex)
536	>	if self.debug :
537	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
538	>	dbgmsg += '\t'+str(ex.output)+'\n'
539	>	print dbgmsg
540	>	except SizeZeroException, ex:
541	>	msg  = "Problem copying %s file" % filetocopy
542	>	msg += str(ex)
543	>	if self.debug :
544	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
545	>	dbgmsg += '\t'+str(ex.output)+'\n'
546	>	print dbgmsg
547	>	ErCode = '60307'
548	>	## when the client commands are not found (wrong env or really missing)
549	>	except MissingCommand, ex:
550	>	ErCode = '10041'
551	>	msg  = "Problem copying %s file" % filetocopy
552	>	msg += str(ex)
553		if self.debug :
554	<	msg += str(ex.detail)+'\n'
555	<	msg += str(ex.output)+'\n'
556	<	msg += "Problem copying %s file" % filetocopy
554	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
555	>	dbgmsg += '\t'+str(ex.output)+'\n'
556	>	print dbgmsg
557	>	except AuthorizationException, ex:
558		ErCode = '60307'
559	<	if ErCode == '0' and protocol.find('srm') == 0:
559	>	msg  = "Problem copying %s file" % filetocopy
560	>	msg += str(ex)
561	>	if self.debug :
562	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
563	>	dbgmsg += '\t'+str(ex.output)+'\n'
564	>	print dbgmsg
565	>	if ErCode == '0' and protocol.find('srmv') == 0:
566		remote_file_size = -1
567		local_file_size = os.path.getsize( source_file )
568		try:
569	<	remote_file_size = sbi_dest.getSize( dest_file )
569	>	remote_file_size = sbi_dest.getSize( dest_file, opt=option )
570	>	if self.debug : print '\t Check of remote size succeded for file %s\n'%dest_file
571		except TransferException, ex:
572	<	msg = str(ex)
572	>	msg  = "Problem checking the size of %s file" % filetocopy
573	>	msg += str(ex)
574		if self.debug :
575	<	msg += str(ex.detail)+'\n'
576	<	msg += str(ex.output)+'\n'
577	<	msg += "Problem checking the size of %s file" % filetocopy
575	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
576	>	dbgmsg += '\t'+str(ex.output)+'\n'
577	>	print dbgmsg
578		ErCode = '60307'
579		except WrongOption, ex:
580	<	msg = str(ex)
580	>	msg  = "Problem checking the size of %s file" % filetocopy
581	>	msg += str(ex)
582		if self.debug :
583	<	msg += str(ex.detail)+'\n'
584	<	msg += str(ex.output)+'\n'
585	<	msg += "Problem checking the size of %s file" % filetocopy
583	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
584	>	dbgmsg += '\t'+str(ex.output)+'\n'
585	>	print dbgmsg
586		ErCode = '60307'
587		if local_file_size != remote_file_size:
588		msg = "File size dosn't match: local size = %s ; remote size = %s " % (local_file_size, remote_file_size)
589		ErCode = '60307'
590
591	+	if ErCode != '0':
592	+	try :
593	+	self.removeFile( sbi_dest, dest_file, option )
594	+	except Exception, ex:
595	+	msg += '\n'+str(ex)
596		return ErCode, msg
597
598	<	def backup(self):
599	<	"""
600	<	Check infos from TFC using existing api obtaining:
601	<	1)destination
602	<	2)protocol
603	<	"""
604	<	return
598	>	def removeFile( self, sbi_dest, filetocopy, option ):
599	>	"""
600	>	"""
601	>	if self.debug : print 'removeFile():\n'
602	>	f_tocopy=filetocopy
603	>	if self.dest_prot != 'local':f_tocopy = os.path.basename(filetocopy)
604	>	try:
605	>	sbi_dest.delete( f_tocopy, opt=option )
606	>	if self.debug : '\t deletion of file %s succeeded\n'%str(filetocopy)
607	>	except OperationException, ex:
608	>	msg  ='ERROR: problems removing partially staged file %s'%filetocopy
609	>	msg += str(ex)
610	>	if self.debug :
611	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
612	>	dbgmsg += '\t'+str(ex.output)+'\n'
613	>	print dbgmsg
614	>	raise Exception(msg)
615	>
616	>	return
617
618		def updateReport(self, file, erCode, reason, lfn='', se='' ):
619		"""
#	Line 394 | Line 637 | class cmscp:
637		cmscp_exit_status = 0
638		txt = ''
639		for file, dict in results.iteritems():
640	+	reason = str(dict['reason'])
641	+	if str(reason).find("'") > -1:
642	+	reason = " ".join(reason.split("'"))
643	+	reason="'%s'"%reason
644		if file:
645		if dict['lfn']=='':
646	<	lfn = '$LFNBaseName/'+os.path.basename(file)
646	>	lfn = '${LFNBaseName}'+os.path.basename(file)
647		se  = '$SE'
648	+	LFNBaseName = '$LFNBaseName'
649		else:
650		lfn = dict['lfn']+os.path.basename(file)
651		se = dict['se']
652	+	LFNBaseName = os.path.dirname(lfn)
653	+	if (LFNBaseName[-1] != '/'):
654	+	LFNBaseName = LFNBaseName + '/'
655		#dict['lfn'] # to be implemented
656	<	txt +=  'echo "Report for File: '+file+'"\n'
657	<	txt +=  'echo "LFN: '+lfn+'"\n'
658	<	txt +=  'echo "StorageElement: '+se+'"\n'
659	<	txt += 'echo "StageOutExitStatusReason ='+dict['reason']+'" | tee -a $RUNTIME_AREA/$repo\n'
656	>	txt += 'echo "Report for File: '+file+'"\n'
657	>	txt += 'echo "LFN: '+lfn+'"\n'
658	>	txt += 'echo "StorageElement: '+se+'"\n'
659	>	txt += 'echo "StageOutExitStatusReason = %s" | tee -a $RUNTIME_AREA/$repo\n'%reason
660		txt += 'echo "StageOutSE = '+se+'" >> $RUNTIME_AREA/$repo\n'
661	+	txt += 'export LFNBaseName='+LFNBaseName+'\n'
662	+	txt += 'export SE='+se+'\n'
663	+
664	+	txt += 'export endpoint='+self.params['destination']+'\n'
665	+
666		if dict['erCode'] != '0':
667		cmscp_exit_status = dict['erCode']
412	–	cmscp_exit_status = dict['erCode']
668		else:
669	<	cmscp_exit_status = dict['erCode']
669	>	txt += 'echo "StageOutExitStatusReason = %s" | tee -a $RUNTIME_AREA/$repo\n'%reason
670		cmscp_exit_status = dict['erCode']
671		txt += '\n'
672		txt += 'export StageOutExitStatus='+str(cmscp_exit_status)+'\n'
673	<	txt +=  'echo "StageOutExitStatus = '+str(cmscp_exit_status)+'" | tee -a $RUNTIME_AREA/$repo\n'
673	>	txt += 'echo "StageOutExitStatus = '+str(cmscp_exit_status)+'" | tee -a $RUNTIME_AREA/$repo\n'
674		outFile.write(str(txt))
675		outFile.close()
676		return
#	Line 424 | Line 679 | class cmscp:
679		def usage():
680
681		msg="""
682	<	required parameters:
683	<	--source        :: REMOTE           :
684	<	--destination   :: REMOTE           :
430	<	--debug             :
431	<	--inFile :: absPath : or name NOT RELATIVE PATH
432	<	--outFIle :: onlyNAME : NOT YET SUPPORTED
682	>	cmscp:
683	>	safe copy of local file  to/from remote SE via lcg_cp/srmcp,
684	>	including success checking  version also for CAF using rfcp command to copy the output to SE
685
686	<	optional parameters
686	>	accepted parameters:
687	>	source           =
688	>	destination      =
689	>	inputFileList    =
690	>	outputFileList   =
691	>	protocol         =
692	>	option           =
693	>	middleware       =
694	>	srm_version      =
695	>	destinationDir   =
696	>	lfn=             =
697	>	local_stage      =  activate stage fall back
698	>	debug            =  activate verbose print out
699	>	help             =  print on line man and exit
700	>
701	>	mandatory:
702	>	* "source" and/or "destination" must always be defined
703	>	* either "middleware" or "protocol" must always be defined
704	>	* "inputFileList" must always be defined
705	>	* if "local_stage" = 1 also  "lfn" must be defined
706		"""
707		print msg
708
#	Line 459 | Line 730 | if __name__ == '__main__' :
730		import getopt
731
732		allowedOpt = ["source=", "destination=", "inputFileList=", "outputFileList=", \
733	<	"protocol=","option=", "middleware=", "srm_version=", "debug", "help"]
733	>	"protocol=","option=", "middleware=", "srm_version=", \
734	>	"destinationDir=", "lfn=", "local_stage", "debug", "help"]
735		try:
736		opts, args = getopt.getopt( sys.argv[1:], "", allowedOpt )
737		except getopt.GetoptError, err:

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines