ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/cmscp.py

Comparing COMP/CRAB/python/cmscp.py (file contents):
Revision 1.43 by fanzago, Mon Feb 16 23:24:36 2009 UTC vs.
Revision 1.56.2.3 by spiga, Tue Sep 29 16:16:12 2009 UTC

#	Line 1 | Line 1
1	<	##!/usr/bin/env python
2	<
1	>	#!/usr/bin/env python
2		import sys, os
3		from ProdCommon.Storage.SEAPI.SElement import SElement, FullPath
4		from ProdCommon.Storage.SEAPI.SBinterface import *
#	Line 10 | Line 9 | class cmscp:
9		def __init__(self, args):
10		"""
11		cmscp
12	<
14	<	safe copy of local file in current directory to remote SE via lcg_cp/srmcp,
12	>	safe copy of local file  to/from remote SE via lcg_cp/srmcp,
13		including success checking  version also for CAF using rfcp command to copy the output to SE
14		input:
15		$1 middleware (CAF, LSF, LCG, OSG)
#	Line 30 | Line 28 | class cmscp:
28		self.params = {"source":'', "destination":'','destinationDir':'', "inputFileList":'', "outputFileList":'', \
29		"protocol":'', "option":'', "middleware":'', "srm_version":'srmv2', "lfn":'' }
30		self.debug = 0
31	<
31	>	self.local_stage = 0
32		self.params.update( args )
33
34		return
#	Line 41 | Line 39 | class cmscp:
39		"""
40		if 'help' in self.params.keys(): HelpOptions()
41		if 'debug' in self.params.keys(): self.debug = 1
42	<	if 'backup' in self.params.keys(): self.backup = 1
42	>	if 'local_stage' in self.params.keys(): self.local_stage = 1
43
44		# source and dest cannot be undefined at same time
45		if not self.params['source']  and not self.params['destination'] :
#	Line 61 | Line 59 | class cmscp:
59		file_to_copy.append(self.params['inputFileList'])
60		self.params['inputFileList'] = file_to_copy
61
62	<
65	<	if not self.params['lfn'] : HelpOptions()
62	>	if not self.params['lfn'] and self.local_stage == 1 : HelpOptions()
63
64		## TO DO:
65		#### add check for outFiles
#	Line 85 | Line 82 | class cmscp:
82		results = self.copy(self.params['inputFileList'], self.params['protocol'], self.params['option'] )
83		return results
84
85	+	def checkLcgUtils( self ):
86	+	"""
87	+	_checkLcgUtils_
88	+	check the lcg-utils version and report
89	+	"""
90	+	import commands
91	+	cmd = "lcg-cp --version | grep lcg_util"
92	+	status, output = commands.getstatusoutput( cmd )
93	+	num_ver = -1
94	+	if output.find("not found") == -1 or status == 0:
95	+	temp = output.split("-")
96	+	version = ""
97	+	if len(temp) >= 2:
98	+	version = output.split("-")[1]
99	+	temp = version.split(".")
100	+	if len(temp) >= 1:
101	+	num_ver = int(temp[0])*10
102	+	num_ver += int(temp[1])
103	+	return num_ver
104	+
105		def setProtocol( self, middleware ):
106		"""
107		define the allowed potocols based on $middlware
#	Line 97 | Line 114 | class cmscp:
114
115		lcgOpt={'srmv1':'-b -D srmv1  -t 2400 --verbose',
116		'srmv2':'-b -D srmv2  -t 2400 --verbose'}
117	+	if self.checkLcgUtils() >= 17:
118	+	lcgOpt={'srmv1':'-b -D srmv1 --srm-timeout 2400 --sendreceive-timeout 240 --connect-timeout 240 --verbose',
119	+	'srmv2':'-b -D srmv2 --srm-timeout 2400 --sendreceive-timeout 240 --connect-timeout 240 --verbose'}
120	+
121		srmOpt={'srmv1':' -report ./srmcp.report -retry_timeout 480000 -retry_num 3 -streams_num=1 ',
122		'srmv2':' -report=./srmcp.report -retry_timeout=480000 -retry_num=3 '}
123		rfioOpt=''
#	Line 107 | Line 128 | class cmscp:
128		(self.params['srm_version'],srmOpt[self.params['srm_version']])]
129		elif middleware.lower() in ['lsf','caf']:
130		supported_protocol = [('rfio',rfioOpt)]
131	+	elif middleware.lower() in ['arc']:
132	+	supported_protocol = [('srmv2','-debug'),('srmv1','-debug')]
133		else:
134		## here we can add support for any kind of protocol,
135		## maybe some local schedulers need something dedicated
#	Line 119 | Line 142 | class cmscp:
142		Checks the status of copy and update result dictionary
143		"""
144		list_retry = []
145	+	list_retry_localSE = []
146		list_not_existing = []
147		list_ok = []
148
#	Line 132 | Line 156 | class cmscp:
156		dict['reason'] = reason
157		elif er_code == '60302':
158		list_not_existing.append( file )
159	<	else:
159	>	elif er_code == '10041':
160		list_retry.append( file )
161	+	## WHAT TO DO IN GENERAL FAILURE CONDITION
162	+	else:
163	+	list_retry_localSE.append( file )
164
165		if self.debug:
166		print "\t file %s \n"%file
#	Line 155 | Line 182 | class cmscp:
182		msg += '\tCopy of %s failed using %s : files not found \n'%(str(list_not_existing),prot)
183		if self.debug : print msg
184
185	<	return copy_results, list_ok, list_retry
185	>	return copy_results, list_ok, list_retry, list_retry_localSE
186
187	<	def backupCopy(self, list_retry, results):
187	>	def LocalCopy(self, list_retry, results):
188		"""
189	<	Tries the backup copy of output to the CloseSE
189	>	Tries the stage out to the CloseSE
190		"""
191		if self.debug:
192	<	print 'in backup() :\n'
192	>	print 'in LocalCopy() :\n'
193		print '\t list_retry %s utils \n'%list_retry
194		print '\t len(list_retry) %s \n'%len(list_retry)
195
#	Line 205 | Line 232 | class cmscp:
232		print '\t surl %s \n'%surl
233
234		destination=os.path.dirname(file_backup[0])
235	+	### FEDE added check for final /
236	+	if ( destination[-1] != '/' ) : destination = destination + '/'
237	+	#####################################
238		self.params['destination']=destination
239
240		if self.debug:
#	Line 213 | Line 243 | class cmscp:
243		print "\t self.params['option']%s \n"%self.params['option']
244
245		for prot, opt in self.setProtocol( self.params['middleware'] ):
246	<	if self.debug: print '\tIn backup trying the stage out with %s utils \n'%prot
247	<	backup_results = self.copy( self.params['inputFileList'], prot, opt )
248	<	if backup_results.keys() == [''] or backup_results.keys() == '' :
249	<	results.update(backup_results)
246	>	if self.debug: print '\tIn LocalCopy trying the stage out with %s utils \n'%prot
247	>	localCopy_results = self.copy( self.params['inputFileList'], prot, opt )
248	>	if localCopy_results.keys() == [''] or localCopy_results.keys() == '' :
249	>	results.update(localCopy_results)
250		else:
251	<	backup_results, list_ok, list_retry = self.checkCopy(backup_results, len(list_files), prot, self.params['lfn'], seName)
252	<	results.update(backup_results)
251	>	localCopy_results, list_ok, list_retry, list_retry_localSE = self.checkCopy(localCopy_results, len(list_files), prot, self.params['lfn'], seName)
252	>	results.update(localCopy_results)
253		if len(list_ok) == len(list_files) :
254		break
255		if len(list_retry):
256		list_files = list_retry
257		else: break
258		if self.debug:
259	<	print "\t backup_results = %s \n"%backup_results
259	>	print "\t localCopy_results = %s \n"%localCopy_results
260
261		return results
262
#	Line 238 | Line 268 | class cmscp:
268		if self.debug:
269		print 'stager() :\n'
270		print '\tmiddleware %s\n'%middleware
271	<	print '\list_files %s\n'%list_files
271	>	print '\tlist_files %s\n'%list_files
272
273		results={}
274		for prot, opt in self.setProtocol( middleware ):
#	Line 247 | Line 277 | class cmscp:
277		if copy_results.keys() == [''] or copy_results.keys() == '' :
278		results.update(copy_results)
279		else:
280	<	copy_results, list_ok, list_retry = self.checkCopy(copy_results, len(list_files), prot)
280	>	copy_results, list_ok, list_retry, list_retry_localSE = self.checkCopy(copy_results, len(list_files), prot)
281		results.update(copy_results)
282		if len(list_ok) == len(list_files) :
283		break
#	Line 255 | Line 285 | class cmscp:
285		list_files = list_retry
286		else: break
287
288	<	if self.backup:
289	<	if len(list_retry):
290	<	results = self.backupCopy(list_retry, results)
288	>	if self.local_stage:
289	>	if len(list_retry_localSE):
290	>	results = self.LocalCopy(list_retry_localSE, results)
291
292		if self.debug:
293		print "\t results %s \n"%results
#	Line 300 | Line 330 | class cmscp:
330		if Destination_SE.protocol in ['gridftp','rfio','srmv2']:
331		try:
332		self.createDir( Destination_SE, Destination_SE.protocol )
333	<	except Exception, ex:
333	>	except OperationException, ex:
334		return self.updateReport('', '60316', str(ex))
335	+	## when the client commands are not found (wrong env or really missing)
336	+	except MissingCommand, ex:
337	+	msg = "ERROR %s %s" %(str(ex), str(ex.detail))
338	+	return self.updateReport('', '10041', msg)
339
340		## prepare for real copy  ##
341		try :
#	Line 320 | Line 354 | class cmscp:
354		try :
355		ErCode, msg = self.checkFileExist( sbi_source, sbi_dest, filetocopy, options )
356		except Exception, ex:
357	<	ErCode = -1
357	>	ErCode = '60307'
358		msg = str(ex)
359		if ErCode == '0':
360		ErCode, msg = self.makeCopy( sbi, filetocopy , options, protocol,sbi_dest )
#	Line 411 | Line 445 | class cmscp:
445		msg += str(ex)
446		if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n'
447		raise Exception(msg)
448	+	## when the client commands are not found (wrong env or really missing)
449	+	except MissingCommand, ex:
450	+	ErCode = '10041'
451	+	msg = "ERROR %s %s" %(str(ex), str(ex.detail))
452	+	return ErCode, msg
453		if not checkSource :
454		ErCode = '60302'
455		msg = "ERROR file %s do not exist"%os.path.basename(filetocopy)
#	Line 440 | Line 479 | class cmscp:
479		msg += str(ex)
480		if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n'
481		raise Exception(msg)
482	+	## when the client commands are not found (wrong env or really missing)
483	+	except MissingCommand, ex:
484	+	ErCode = '10041'
485	+	msg = "ERROR %s %s" %(str(ex), str(ex.detail))
486	+	return ErCode, msg
487		if check :
488		ErCode = '60303'
489		msg = "file %s already exist"%os.path.basename(filetocopy)
#	Line 466 | Line 510 | class cmscp:
510		ErCode = '0'
511		msg = ''
512
513	+	if  self.params['option'].find('space_token'):
514	+	space_tocken=self.params['option'].split('=')[1]
515	+	if protocol == 'srmv2': option = '%s -space_tocken=%s'%(option,space_tocken)
516	+	if protocol == 'srm-lcg': option = '%s -S %s'%(option,space_tocken)
517		try:
518		sbi.copy( source_file , dest_file , opt = option)
519		except TransferException, ex:
#	Line 482 | Line 530 | class cmscp:
530		if self.debug :
531		dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
532		dbgmsg += '\t'+str(ex.output)+'\n'
533	<	print dbsmsg
533	>	print dbgmsg
534	>	except SizeZeroException, ex:
535	>	msg  = "Problem copying %s file" % filetocopy
536	>	msg += str(ex)
537	>	if self.debug :
538	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
539	>	dbgmsg += '\t'+str(ex.output)+'\n'
540	>	print dbgmsg
541		ErCode = '60307'
542	+	## when the client commands are not found (wrong env or really missing)
543	+	except MissingCommand, ex:
544	+	ErCode = '10041'
545	+	msg  = "Problem copying %s file" % filetocopy
546	+	msg += str(ex)
547	+	if self.debug :
548	+	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
549	+	dbgmsg += '\t'+str(ex.output)+'\n'
550	+	print dbgmsg
551	+	except AuthorizationException, ex:
552	+	ErCode = '60307'
553	+	msg  = "Problem copying %s file" % filetocopy
554	+	msg += str(ex)
555	+	if self.debug :
556	+	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
557	+	dbgmsg += '\t'+str(ex.output)+'\n'
558	+	print dbgmsg
559		if ErCode == '0' and protocol.find('srmv') == 0:
560		remote_file_size = -1
561		local_file_size = os.path.getsize( source_file )
#	Line 537 | Line 609 | class cmscp:
609
610		return
611
540	–	def backup(self):
541	–	"""
542	–	Check infos from TFC using existing api obtaining:
543	–	1)destination
544	–	2)protocol
545	–	"""
546	–	return
547	–
612		def updateReport(self, file, erCode, reason, lfn='', se='' ):
613		"""
614		Update the final stage out infos
#	Line 567 | Line 631 | class cmscp:
631		cmscp_exit_status = 0
632		txt = ''
633		for file, dict in results.iteritems():
634	+	reason = str(dict['reason'])
635	+	if str(reason).find("'") > -1:
636	+	reason = " ".join(reason.split("'"))
637	+	reason="'%s'"%reason
638		if file:
639		if dict['lfn']=='':
640		lfn = '$LFNBaseName/'+os.path.basename(file)
#	Line 574 | Line 642 | class cmscp:
642		else:
643		lfn = dict['lfn']+os.path.basename(file)
644		se = dict['se']
577	–
645		#dict['lfn'] # to be implemented
646		txt += 'echo "Report for File: '+file+'"\n'
647		txt += 'echo "LFN: '+lfn+'"\n'
648		txt += 'echo "StorageElement: '+se+'"\n'
649	<	txt += 'echo "StageOutExitStatusReason ='+dict['reason']+'" | tee -a $RUNTIME_AREA/$repo\n'
649	>	txt += 'echo "StageOutExitStatusReason = %s" | tee -a $RUNTIME_AREA/$repo\n'%reason
650		txt += 'echo "StageOutSE = '+se+'" >> $RUNTIME_AREA/$repo\n'
651		#txt += 'export LFNBaseName='+lfn+'\n'
652		txt += 'export SE='+se+'\n'
653	+	### FEDE per CopyData ####
654	+
655	+	txt += 'export endpoint='+self.params['destination']+'\n'
656
657		if dict['erCode'] != '0':
658		cmscp_exit_status = dict['erCode']
659		else:
660	<	txt += 'echo "StageOutExitStatusReason ='+dict['reason']+'" | tee -a $RUNTIME_AREA/$repo\n'
660	>	txt += 'echo "StageOutExitStatusReason = %s" | tee -a $RUNTIME_AREA/$repo\n'%reason
661		cmscp_exit_status = dict['erCode']
662		cmscp_exit_status = dict['erCode']
663		txt += '\n'
#	Line 601 | Line 671 | class cmscp:
671		def usage():
672
673		msg="""
674	<	required parameters:
675	<	--source        :: REMOTE           :
676	<	--destination   :: REMOTE           :
607	<	--debug             :
608	<	--inFile :: absPath : or name NOT RELATIVE PATH
609	<	--outFIle :: onlyNAME : NOT YET SUPPORTED
674	>	cmscp:
675	>	safe copy of local file  to/from remote SE via lcg_cp/srmcp,
676	>	including success checking  version also for CAF using rfcp command to copy the output to SE
677
678	<	optional parameters
678	>	accepted parameters:
679	>	source           =
680	>	destination      =
681	>	inputFileList    =
682	>	outputFileList   =
683	>	protocol         =
684	>	option           =
685	>	middleware       =
686	>	srm_version      =
687	>	destinationDir   =
688	>	lfn=             =
689	>	local_stage      =  activate stage fall back
690	>	debug            =  activate verbose print out
691	>	help             =  print on line man and exit
692	>
693	>	mandatory:
694	>	* "source" and/or "destination" must always be defined
695	>	* either "middleware" or "protocol" must always be defined
696	>	* "inputFileList" must always be defined
697	>	* if "local_stage" = 1 also  "lfn" must be defined
698		"""
699		print msg
700
#	Line 637 | Line 723 | if __name__ == '__main__' :
723
724		allowedOpt = ["source=", "destination=", "inputFileList=", "outputFileList=", \
725		"protocol=","option=", "middleware=", "srm_version=", \
726	<	"destinationDir=","debug", "help", "lfn="]
726	>	"destinationDir=", "lfn=", "local_stage", "debug", "help"]
727		try:
728		opts, args = getopt.getopt( sys.argv[1:], "", allowedOpt )
729		except getopt.GetoptError, err:

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines