ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/cmscp.py

Comparing COMP/CRAB/python/cmscp.py (file contents):
Revision 1.6 by spiga, Sun Sep 28 21:17:05 2008 UTC vs.
Revision 1.20 by spiga, Tue Oct 28 17:40:55 2008 UTC

#	Line 1 | Line 1
1		#!/usr/bin/env python
2
3	<	import sys, getopt, string
4	<	import os, popen2
5	<	from ProdCommon.Storage.SEAPI.SElement import SElement, FullPath
3	>	import sys, os
4	>	from ProdCommon.Storage.SEAPI.SElement import SElement, FullPath
5		from ProdCommon.Storage.SEAPI.SBinterface import *
6	<
6	>	from ProdCommon.Storage.SEAPI.Exceptions import *
7
8
9		class cmscp:
10	<	def __init__(self, argv):
10	>	def __init__(self, args):
11		"""
12		cmscp
13
14	<	safe copy of local file in current directory to remote SE via lcg_cp/srmcp,
14	>	safe copy of local file in current directory to remote SE via lcg_cp/srmcp,
15		including success checking  version also for CAF using rfcp command to copy the output to SE
16		input:
17		$1 middleware (CAF, LSF, LCG, OSG)
18		$2 local file (the absolute path of output file or just the name if it's in top dir)
19		$3 if needed: file name (the output file name)
20		$5 remote SE (complete endpoint)
21	<	$6 srm version
21	>	$6 srm version
22		output:
23		return 0 if all ok
24		return 60307 if srmcp failed
25		return 60303 if file already exists in the SE
26		"""
27	+
28		#set default
29	+	self.params = {"source":'', "destination":'', "inputFileList":'', "outputFileList":'', \
30	+	"protocol":'', "option":'', "middleware":'', "srm_version":'srmv2'}
31		self.debug = 0
30	–	self.source = ''
31	–	self.destination = ''
32	–	self.file_to_copy = []
33	–	self.remote_file_name = []
34	–	self.protocol = ''
35	–	self.middleware = ''
36	–	self.srmv = ''
37	–
38	–	# default for stage out approach To be used with "middleware"
39	–	self.lcgOpt='-b -D srmv2 --vo cms -t 2400 --verbose'
40	–	self.srmOpt='-debug=true -report ./srmcp.report -retry_timeout 480000 -retry_num 3'
41	–	self.rfioOpt=''
42	–	# default to be used with protocol
43	–	self.opt=''
32
33	<	try:
34	<	opts, args = getopt.getopt(argv, "", ["source=", "destination=", "inputFileList=", "outputFileList=", \
47	<	"protocol=", "middleware=", "srm_version=", "debug", "help"])
48	<	except getopt.GetoptError:
49	<	print self.usage()
50	<	sys.exit(2)
51	<
52	<	self.setAndCheck(opts)
53	<
33	>	self.params.update( args )
34	>
35		return
36
37	<	def setAndCheck( self, opts ):
37	>	def processOptions( self ):
38		"""
39	<	Set and check command line parameter
39	>	check command line parameter
40		"""
41	<	if not opts :
42	<	print self.usage()
43	<	sys.exit()
44	<	for opt, arg in opts :
45	<	if opt  == "--help" :
46	<	print self.usage()
47	<	sys.exit()
48	<	elif opt == "--debug" :
49	<	self.debug = 1
50	<	elif opt == "--source" :
51	<	self.source = arg
52	<	elif opt == "--destination":
53	<	self.destination = arg
54	<	elif opt == "--inputFileList":
74	<	infile = arg
75	<	elif opt == "--outputFileList":
76	<	out_file
77	<	elif opt == "--protocol":
78	<	self.protocol = arg
79	<	elif opt == "--middleware":
80	<	self.middleware = arg
81	<	elif opt == "--srm_version":
82	<	self.srmv = arg
83	<
84	<	# source and dest cannot be undefined at same time
85	<	if self.source == '' and self.destination == '':
86	<	print self.usage()
87	<	sys.exit()
88	<	# if middleware is not defined --> protocol cannot be empty
89	<	if self.middleware == '' and self.protocol == '':
90	<	print self.usage()
91	<	sys.exit()
92	<	# input file must be defined
93	<	if infile == '':
94	<	print self.usage()
95	<	sys.exit()
41	>
42	>	if 'help' in self.params.keys(): HelpOptions()
43	>	if 'debug' in self.params.keys(): self.debug = 1
44	>
45	>	# source and dest cannot be undefined at same time
46	>	if not self.params['source']  and not self.params['destination'] :
47	>	HelpOptions()
48	>
49	>	# if middleware is not defined --> protocol cannot be empty
50	>	if not self.params['middleware'] and not self.params['protocol'] :
51	>	HelpOptions()
52	>
53	>	# input file must be defined
54	>	if not self.params['inputFileList'] : HelpOptions()
55		else:
56	<	if infile.find(','):
57	<	[self.file_to_copy.append(x.strip()) for x in infile.split(',')]
58	<	else:
59	<	self.file_to_copy.append(infile)
60	<
56	>	file_to_copy=[]
57	>	if self.params['inputFileList'].find(','):
58	>	[file_to_copy.append(x.strip()) for x in self.params['inputFileList'].split(',')]
59	>	else:
60	>	file_to_copy.append(self.params['inputFileList'])
61	>	self.params['inputFileList'] = file_to_copy
62	>
63		## TO DO:
64		#### add check for outFiles
65		#### add map {'inFileNAME':'outFileNAME'} to change out name
66
67		return
68
69	<	def run( self ):
69	>	def run( self ):
70		"""
71	<	Check if running on UI (no $middleware) or
72	<	on WN (on the Grid), and take different action
71	>	Check if running on UI (no $middleware) or
72	>	on WN (on the Grid), and take different action
73		"""
113	–	if self.middleware :
114	–	results = self.stager()
115	–	else:
116	–	results = self.copy( self.file_to_copy, self.protocol , self.opt)
74
75	<	self.finalReport(results,self.middleware)
75	>	self.processOptions()
76	>	# stage out from WN
77	>	if self.params['middleware'] :
78	>	results = self.stager(self.params['middleware'],self.params['inputFileList'])
79	>	self.finalReport(results)
80	>	# Local interaction with SE
81	>	else:
82	>	results = self.copy(self.params['inputFilesList'], self.params['protocol'], self.params['option'] )
83	>	return results
84
85	<	return
121	<
122	<	def setProtocol( self ):
85	>	def setProtocol( self, middleware ):
86		"""
87		define the allowed potocols based on $middlware
88	<	which depend on scheduler
88	>	which depend on scheduler
89		"""
90	<	if self.middleware.lower() in ['osg','lcg']:
91	<	supported_protocol = ['srm-lcg','srmv2']
92	<	self.OptMap = {'srm-lcg': self.lcgOpt,
93	<	'srmv2': self.srmOpt }
94	<	elif self.middleware.lower() in ['lsf','caf']:
95	<	supported_protocol = ['rfio']
96	<	self.OptMap = {'rfio': self.rfioOpt }
90	>	# default To be used with "middleware"
91	>	lcgOpt={'srmv1':'-b -D srmv1  -t 2400 --verbose',
92	>	'srmv2':'-b -D srmv2  -t 2400 --verbose'}
93	>	srmOpt={'srmv1':' -report ./srmcp.report -retry_timeout 480000 -retry_num 3 -streams_num=1 ',
94	>	'srmv2':' -report ./srmcp.report -retry_timeout 480000 -retry_num 3 '}
95	>	rfioOpt=''
96	>
97	>	supported_protocol = None
98	>	if middleware.lower() in ['osg','lcg','condor']:
99	>	supported_protocol = [('srm-lcg',lcgOpt[self.params['srm_version']]),\
100	>	(self.params['srm_version'],srmOpt[self.params['srm_version']])]
101	>	elif middleware.lower() in ['lsf','caf']:
102	>	supported_protocol = [('rfio',rfioOpt)]
103		else:
104	<	## here we can add support for any kind of protocol,
104	>	## here we can add support for any kind of protocol,
105		## maybe some local schedulers need something dedicated
106		pass
107		return supported_protocol
108
109	<	def stager( self ):
109	>	def stager( self, middleware, list_files ):
110		"""
111		Implement the logic for remote stage out
112		"""
144	–	protocols = self.setProtocol()
145	–	count=0
146	–	list_files = self.file_to_copy
113		results={}
114	<	for prot in protocols:
115	<	if self.debug: print 'Trying stage out with %s utils \n'%prot
116	<	copy_results = self.copy( list_files, prot, self.OptMap[prot] )
117	<	list_retry = []
118	<	list_existing = []
119	<	list_ok = []
120	<	for file, dict in copy_results.iteritems():
121	<	er_code = dict['erCode']
156	<	if er_code == '60307': list_retry.append( file )
157	<	elif er_code == '60303': list_existing.append( file )
158	<	else:
159	<	list_ok.append(file)
160	<	reason = 'Copy succedeed with %s utils'%prot
161	<	upDict = self.updateReport(file, er_code, reason)
162	<	copy_results.update(upDict)
163	<	results.update(copy_results)
164	<	if len(list_ok) != 0:
165	<	msg = 'Copy of %s succedeed with %s utils\n'%(str(list_ok),prot)
166	<	# print msg
167	<	if len(list_ok) == len(list_files) :
168	<	break
114	>	for prot, opt in self.setProtocol( middleware ):
115	>	if self.debug: print 'Trying stage out with %s utils \n'%prot
116	>	copy_results = self.copy( list_files, prot, opt )
117	>	list_retry = []
118	>	list_existing = []
119	>	list_ok = []
120	>	if copy_results.keys() == '':
121	>	results.update(copy_results)
122		else:
123	<	#       print 'Copy of files %s failed using %s...\n'%(str(list_retry)+str(list_existing),prot)
124	<	if len(list_retry): list_files = list_retry
125	<	else: break
126	<	count =+1
127	<
128	<	#### TODO Daniele
123	>	for file, dict in copy_results.iteritems():
124	>	er_code = dict['erCode']
125	>	if er_code == '0':
126	>	list_ok.append(file)
127	>	reason = 'Copy succedeed with %s utils'%prot
128	>	upDict = self.updateReport(file, er_code, reason)
129	>	copy_results.update(upDict)
130	>	elif er_code == '60303': list_existing.append( file )
131	>	else: list_retry.append( file )
132	>	results.update(copy_results)
133	>	if len(list_ok) != 0:
134	>	msg = 'Copy of %s succedeed with %s utils\n'%(str(list_ok),prot)
135	>	if self.debug : print msg
136	>	if len(list_ok) == len(list_files) :
137	>	break
138	>	else:
139	>	if self.debug : print 'Copy of files %s failed using %s...\n'%(str(list_retry)+str(list_existing),prot)
140	>	if len(list_retry): list_files = list_retry
141	>	else: break
142	>
143	>	#### TODO Daniele
144		#check is something fails and created related dict
145	<	#      backup = self.analyzeResults(results)
146	<
147	<	#      if backup :
145	>	#      backup = self.analyzeResults(results)
146	>
147	>	#      if backup :
148		#          msg = 'WARNING: backup logic is under implementation\n'
149		#          #backupDict = self.backup()
150	<	#          ### NOTE: IT MUST RETURN a DICT contains also LFN and SE Name
150	>	#          ### NOTE: IT MUST RETURN a DICT contains also LFN and SE Name
151		#          results.update(backupDict)
152		#          print msg
153		return results
154
155		def initializeApi(self, protocol ):
156		"""
157	<	Instantiate storage interface
157	>	Instantiate storage interface
158		"""
159	<	source_prot = protocol
160	<	dest_prot = protocol
161	<	if self.source == '' : source_prot = 'local'
162	<	Source_SE  = self.storageInterface( self.source, source_prot )
163	<	if self.destination == '' : dest_prot = 'local'
164	<	Destination_SE = self.storageInterface( self.destination, dest_prot )
159	>	self.source_prot = protocol
160	>	self.dest_prot = protocol
161	>	if not self.params['source'] : self.source_prot = 'local'
162	>	Source_SE  = self.storageInterface( self.params['source'], self.source_prot )
163	>	if not self.params['destination'] : self.dest_prot = 'local'
164	>	Destination_SE = self.storageInterface( self.params['destination'], self.dest_prot )
165
166		if self.debug :
167	<	print '(source=%s,  protocol=%s)'%(self.source, source_prot)
168	<	print '(destination=%s,  protocol=%s)'%(self.destination, dest_prot)
167	>	print '(source=%s,  protocol=%s)'%(self.params['source'], self.source_prot)
168	>	print '(destination=%s,  protocol=%s)'%(self.params['destination'], self.dest_prot)
169
170		return Source_SE, Destination_SE
171
172	<	def copy( self, list_file, protocol, opt):
172	>	def copy( self, list_file, protocol, options ):
173		"""
174	<	Make the real file copy using SE API
174	>	Make the real file copy using SE API
175		"""
176		if self.debug :
177	<	print 'copy(): using %s protocol'%protocol
178	<	Source_SE, Destination_SE = self.initializeApi( protocol )
177	>	print 'copy(): using %s protocol'%protocol
178	>	try:
179	>	Source_SE, Destination_SE = self.initializeApi( protocol )
180	>	except Exception, ex:
181	>	return self.updateReport('', '-1', str(ex))
182
183	<	# create remote dir
183	>	# create remote dir
184		if protocol in ['gridftp','rfio']:
185	<	self.createDir( Destination_SE, protocol )
185	>	try:
186	>	self.createDir( Destination_SE, protocol )
187	>	except Exception, ex:
188	>	return self.updateReport('', '60316', str(ex))
189
190		## prepare for real copy  ##
191	<	sbi = SBinterface( Source_SE, Destination_SE )
192	<	sbi_dest = SBinterface(Destination_SE)
191	>	try :
192	>	sbi = SBinterface( Source_SE, Destination_SE )
193	>	sbi_dest = SBinterface(Destination_SE)
194	>	sbi_source = SBinterface(Source_SE)
195	>	except ProtocolMismatch, ex:
196	>	msg = str(ex)+'\n'
197	>	msg += "ERROR : Unable to create SBinterface with %s protocol\n"%protocol
198	>	return self.updateReport('', '-1', str(ex))
199
200		results = {}
201	<	## loop over the complete list of files
202	<	for filetocopy in list_file:
201	>	## loop over the complete list of files
202	>	for filetocopy in list_file:
203		if self.debug : print 'start real copy for %s'%filetocopy
204	<	ErCode, msg = self.checkFileExist( sbi_dest, os.path.basename(filetocopy) )
205	<	if ErCode == '0':
206	<	ErCode, msg = self.makeCopy( sbi, filetocopy , opt)
207	<	if self.debug : print 'Copy results for %s is %s'%( os.path.basename(filetocopy) ,ErCode)
204	>	try :
205	>	ErCode, msg = self.checkFileExist( sbi_source, sbi_dest, filetocopy )
206	>	except Exception, ex:
207	>	ErCode = -1
208	>	msg = str(ex)
209	>	if ErCode == '0':
210	>	ErCode, msg = self.makeCopy( sbi, filetocopy , options, protocol,sbi_dest )
211	>	if self.debug : print 'Copy results for %s is %s'%( os.path.basename(filetocopy), ErCode)
212		results.update( self.updateReport(filetocopy, ErCode, msg))
213		return results
230	–
231	–	def updateReport(self, file, erCode, reason, lfn='', se='' ):
232	–	"""
233	–	Update the final stage out infos
234	–	"""
235	–	jobStageInfo={}
236	–	jobStageInfo['erCode']=erCode
237	–	jobStageInfo['reason']=reason
238	–	jobStageInfo['lfn']=lfn
239	–	jobStageInfo['se']=se
214
241	–	report = { file : jobStageInfo}
242	–	return report
243	–
244	–	def finalReport( self , results, middleware ):
245	–	"""
246	–	It should return a clear list of LFNs for each SE where data are stored.
247	–	allow "crab -copyLocal" or better "crab -copyOutput". TO_DO.
248	–	"""
249	–	if middleware:
250	–	outFile = open('cmscpReport.sh',"a")
251	–	cmscp_exit_status = 0
252	–	txt = ''
253	–	for file, dict in results.iteritems():
254	–	if dict['lfn']=='':
255	–	lfn = '$LFNBaseName/'+os.path.basename(file)
256	–	se  = '$SE'
257	–	else:
258	–	lfn = dict['lfn']+os.pat.basename(file)
259	–	se = dict['se']
260	–	#dict['lfn'] # to be implemented
261	–	txt +=  'echo "Report for File: '+file+'"\n'
262	–	txt +=  'echo "LFN: '+lfn+'"\n'
263	–	txt +=  'echo "StorageElement: '+se+'"\n'
264	–	txt += 'echo "StageOutExitStatusReason ='+dict['reason']+'" | tee -a $RUNTIME_AREA/$repo\n'
265	–	txt += 'echo "StageOutSE = '+se+'" >> $RUNTIME_AREA/$repo\n'
266	–	if dict['erCode'] != '0':
267	–	cmscp_exit_status = dict['erCode']
268	–	txt += '\n'
269	–	txt += 'export StageOutExitStatus='+str(cmscp_exit_status)+'\n'
270	–	txt +=  'echo "StageOutExitStatus = '+str(cmscp_exit_status)+'" | tee -a $RUNTIME_AREA/$repo\n'
271	–	outFile.write(str(txt))
272	–	outFile.close()
273	–	else:
274	–	for file, code in results.iteritems():
275	–	print 'error code = %s for file %s'%(code,file)
276	–	return
215
216		def storageInterface( self, endpoint, protocol ):
217		"""
218	<	Create the storage interface.
218	>	Create the storage interface.
219		"""
220		try:
221		interface = SElement( FullPath(endpoint), protocol )
222	<	except Exception, ex:
223	<	msg = ''
222	>	except ProtocolUnknown, ex:
223	>	msg = ''
224		if self.debug : msg = str(ex)+'\n'
225	<	msg += "ERROR : Unable to create interface with %s protocol\n"%protocol
226	<	print msg
225	>	msg += "ERROR : Unable to create interface with %s protocol\n"%protocol
226	>	raise Exception(msg)
227
228		return interface
229
292	–	def checkDir(self, Destination_SE, protocol):
293	–	'''
294	–	ToBeImplemented NEEDED for castor
295	–	'''
296	–	return
297	–
230		def createDir(self, Destination_SE, protocol):
231		"""
232	<	Create remote dir for gsiftp/rfio REALLY TEMPORARY
233	<	this should be transparent at SE API level.
232	>	Create remote dir for gsiftp REALLY TEMPORARY
233	>	this should be transparent at SE API level.
234		"""
235	<	ErCode = '0'
304	<	msg_1 = ''
235	>	msg = ''
236		try:
237		action = SBinterface( Destination_SE )
238		action.createDir()
239	<	if self.debug: print "The directory has been created using protocol %s\n"%protocol
240	<	except Exception, ex:
241	<	msg = ''
242	<	if self.debug : msg = str(ex)+'\n'
243	<	msg_1 = "ERROR: problem with the directory creation using %s protocol \n"%protocol
244	<	msg += msg_1
245	<	ErCode = '60316'
246	<	#print msg
239	>	if self.debug: msg+= "The directory has been created using protocol %s\n"%protocol
240	>	except TransferException, ex:
241	>	msg = str(ex)
242	>	if self.debug :
243	>	msg += str(ex.detail)+'\n'
244	>	msg += str(ex.output)+'\n'
245	>	msg += "ERROR: problem with the directory creation using %s protocol \n"%protocol
246	>	raise Exceptions(msg)
247	>	except OperationException, ex:
248	>	msg = str(ex)
249	>	if self.debug : msg += str(ex.detail)+'\n'
250	>	msg += "ERROR: problem with the directory creation using %s protocol \n"%protocol
251
252	<	return ErCode, msg_1
252	>	return msg
253
254	<	def checkFileExist(self, sbi, filetocopy):
254	>	def checkFileExist( self, sbi_source, sbi_dest, filetocopy ):
255	>	"""
256	>	Check both if source file exist AND
257	>	if destination file ALREADY exist.
258		"""
321	–	Check if file to copy already exist
322	–	"""
323	–	try:
324	–	check = sbi.checkExists(filetocopy)
325	–	except Exception, ex:
326	–	msg = ''
327	–	if self.debug : msg = str(ex)+'\n'
328	–	msg += "ERROR: problem with check File Exist using %s protocol \n"%protocol
329	–	# print msg
259		ErCode = '0'
260		msg = ''
261	<	if check :
262	<	ErCode = '60303'
263	<	msg = "file %s already exist"%filetocopy
264	<	print msg
261	>	f_tocopy=filetocopy
262	>	if self.source_prot != 'local':f_tocopy = os.path.basename(filetocopy)
263	>	try:
264	>	checkSource = sbi_source.checkExists( f_tocopy )
265	>	except OperationException, ex:
266	>	msg = str(ex)
267	>	if self.debug :
268	>	msg += str(ex.detail)+'\n'
269	>	msg += str(ex.output)+'\n'
270	>	msg +='ERROR: problems checkig if source file %s exist'%filetocopy
271	>	raise Exception(msg)
272	>	except WrongOption, ex:
273	>	msg = str(ex)
274	>	if self.debug :
275	>	msg += str(ex.detail)+'\n'
276	>	msg += str(ex.output)+'\n'
277	>	msg +='ERROR problems checkig if source file % exist'%filetocopy
278	>	raise Exception(msg)
279	>	if not checkSource :
280	>	ErCode = '60302'
281	>	msg = "ERROR file %s do not exist"%os.path.basename(filetocopy)
282	>	return ErCode, msg
283
284	<	return ErCode,msg
284	>	f_tocopy=filetocopy
285	>	if self.dest_prot != 'local':f_tocopy = os.path.basename(filetocopy)
286	>	try:
287	>	check = sbi_dest.checkExists( f_tocopy )
288	>	except OperationException, ex:
289	>	msg = str(ex)
290	>	if self.debug :
291	>	msg += str(ex.detail)+'\n'
292	>	msg += str(ex.output)+'\n'
293	>	msg +='ERROR: problems checkig if file %s already exist'%filetocopy
294	>	raise Exception(msg)
295	>	except WrongOption, ex:
296	>	msg = str(ex)
297	>	if self.debug :
298	>	msg += str(ex.detail)+'\n'
299	>	msg += str(ex.output)+'\n'
300	>	msg +='ERROR problems checkig if file % already exist'%filetocopy
301	>	raise Exception(msg)
302	>	if check :
303	>	ErCode = '60303'
304	>	msg = "file %s already exist"%os.path.basename(filetocopy)
305
306	<	def makeCopy(self, sbi, filetocopy, opt ):
306	>	return ErCode, msg
307	>
308	>	def makeCopy(self, sbi, filetocopy, option, protocol, sbi_dest ):
309		"""
310	<	call the copy API.
310	>	call the copy API.
311		"""
312	<	path = os.path.dirname(filetocopy)
312	>	path = os.path.dirname(filetocopy)
313		file_name =  os.path.basename(filetocopy)
314		source_file = filetocopy
315		dest_file = file_name ## to be improved supporting changing file name  TODO
316	<	if self.source == '' and path == '':
316	>	if self.params['source'] == '' and path == '':
317		source_file = os.path.abspath(filetocopy)
318	<	elif self.destination =='':
318	>	elif self.params['destination'] =='':
319		dest_file = os.path.join(os.getcwd(),file_name)
320	<	elif self.source != '' and self.destination != '' :
321	<	source_file = file_name
320	>	elif self.params['source'] != '' and self.params['destination'] != '' :
321	>	source_file = file_name
322	>
323		ErCode = '0'
324		msg = ''
325	<
325	>
326		try:
327	<	pippo = sbi.copy( source_file , dest_file , opt = opt)
328	<	if self.protocol == 'srm' : self.checkSize( sbi, filetocopy )
329	<	except Exception, ex:
330	<	msg = ''
331	<	if self.debug : msg = str(ex)+'\n'
332	<	msg = "Problem copying %s file with %s command"%( filetocopy, protocol )
327	>	sbi.copy( source_file , dest_file , opt = option)
328	>	except TransferException, ex:
329	>	msg = str(ex)
330	>	if self.debug :
331	>	msg += str(ex.detail)+'\n'
332	>	msg += str(ex.output)+'\n'
333	>	msg += "Problem copying %s file" % filetocopy
334	>	ErCode = '60307'
335	>	except WrongOption, ex:
336	>	msg = str(ex)
337	>	if self.debug :
338	>	msg += str(ex.detail)+'\n'
339	>	msg += str(ex.output)+'\n'
340	>	msg += "Problem copying %s file" % filetocopy
341		ErCode = '60307'
342	<	#print msg
342	>	if ErCode == '0' and protocol.find('srm') == 0:
343	>	remote_file_size = -1
344	>	local_file_size = os.path.getsize( source_file )
345	>	try:
346	>	remote_file_size = sbi_dest.getSize( dest_file )
347	>	except TransferException, ex:
348	>	msg = str(ex)
349	>	if self.debug :
350	>	msg += str(ex.detail)+'\n'
351	>	msg += str(ex.output)+'\n'
352	>	msg += "Problem checking the size of %s file" % filetocopy
353	>	ErCode = '60307'
354	>	except WrongOption, ex:
355	>	msg = str(ex)
356	>	if self.debug :
357	>	msg += str(ex.detail)+'\n'
358	>	msg += str(ex.output)+'\n'
359	>	msg += "Problem checking the size of %s file" % filetocopy
360	>	ErCode = '60307'
361	>	if local_file_size != remote_file_size:
362	>	msg = "File size dosn't match: local size = %s ; remote size = %s " % (local_file_size, remote_file_size)
363	>	ErCode = '60307'
364
365		return ErCode, msg
366	<
367	<	'''
369	<	def checkSize()
370	<	"""
371	<	Using srm needed a check of the ouptut file size.
372	<	"""
373	<
374	<	echo "--> remoteSize = $remoteSize"
375	<	## for local file
376	<	localSize=$(stat -c%s "$path_out_file")
377	<	echo "-->  localSize = $localSize"
378	<	if [ $localSize != $remoteSize ]; then
379	<	echo "Local fileSize $localSize does not match remote fileSize $remoteSize"
380	<	echo "Copy failed: removing remote file $destination"
381	<	srmrm $destination
382	<	cmscp_exit_status=60307
383	<
384	<
385	<	echo "Problem copying $path_out_file to $destination with srmcp command"
386	<	StageOutExitStatusReason='remote and local file dimension not match'
387	<	echo "StageOutReport = `cat ./srmcp.report`"
388	<	'''
389	<	def backup(self):
366	>
367	>	def backup(self):
368		"""
369		Check infos from TFC using existing api obtaining:
370		1)destination
#	Line 394 | Line 372 | class cmscp:
372		"""
373		return
374
375	<	def usage(self):
375	>	def updateReport(self, file, erCode, reason, lfn='', se='' ):
376	>	"""
377	>	Update the final stage out infos
378	>	"""
379	>	jobStageInfo={}
380	>	jobStageInfo['erCode']=erCode
381	>	jobStageInfo['reason']=reason
382	>	jobStageInfo['lfn']=lfn
383	>	jobStageInfo['se']=se
384	>
385	>	report = { file : jobStageInfo}
386	>	return report
387
388	<	msg="""
389	<	required parameters:
390	<	--source        :: REMOTE           :
391	<	--destination   :: REMOTE           :
403	<	--debug             :
404	<	--inFile :: absPath : or name NOT RELATIVE PATH
405	<	--outFIle :: onlyNAME : NOT YET SUPPORTED
406	<
407	<	optional parameters
388	>	def finalReport( self , results ):
389	>	"""
390	>	It a list of LFNs for each SE where data are stored.
391	>	allow "crab -copyLocal" or better "crab -copyOutput". TO_DO.
392		"""
393	<	return msg
393	>	outFile = open('cmscpReport.sh',"a")
394	>	cmscp_exit_status = 0
395	>	txt = ''
396	>	for file, dict in results.iteritems():
397	>	if file:
398	>	if dict['lfn']=='':
399	>	lfn = '$LFNBaseName/'+os.path.basename(file)
400	>	se  = '$SE'
401	>	else:
402	>	lfn = dict['lfn']+os.path.basename(file)
403	>	se = dict['se']
404	>	#dict['lfn'] # to be implemented
405	>	txt +=  'echo "Report for File: '+file+'"\n'
406	>	txt +=  'echo "LFN: '+lfn+'"\n'
407	>	txt +=  'echo "StorageElement: '+se+'"\n'
408	>	txt += 'echo "StageOutExitStatusReason ='+dict['reason']+'" | tee -a $RUNTIME_AREA/$repo\n'
409	>	txt += 'echo "StageOutSE = '+se+'" >> $RUNTIME_AREA/$repo\n'
410	>	if dict['erCode'] != '0':
411	>	cmscp_exit_status = dict['erCode']
412	>	cmscp_exit_status = dict['erCode']
413	>	else:
414	>	cmscp_exit_status = dict['erCode']
415	>	cmscp_exit_status = dict['erCode']
416	>	txt += '\n'
417	>	txt += 'export StageOutExitStatus='+str(cmscp_exit_status)+'\n'
418	>	txt +=  'echo "StageOutExitStatus = '+str(cmscp_exit_status)+'" | tee -a $RUNTIME_AREA/$repo\n'
419	>	outFile.write(str(txt))
420	>	outFile.close()
421	>	return
422	>
423	>
424	>	def usage():
425	>
426	>	msg="""
427	>	required parameters:
428	>	--source        :: REMOTE           :
429	>	--destination   :: REMOTE           :
430	>	--debug             :
431	>	--inFile :: absPath : or name NOT RELATIVE PATH
432	>	--outFIle :: onlyNAME : NOT YET SUPPORTED
433	>
434	>	optional parameters
435	>	"""
436	>	print msg
437	>
438	>	return
439	>
440	>	def HelpOptions(opts=[]):
441	>	"""
442	>	Check otps, print help if needed
443	>	prepare dict = { opt : value }
444	>	"""
445	>	dict_args = {}
446	>	if len(opts):
447	>	for opt, arg in opts:
448	>	dict_args[opt.split('--')[1]] = arg
449	>	if opt in ('-h','-help','--help') :
450	>	usage()
451	>	sys.exit(0)
452	>	return dict_args
453	>	else:
454	>	usage()
455	>	sys.exit(0)
456
457		if __name__ == '__main__' :
458	+
459	+	import getopt
460	+
461	+	allowedOpt = ["source=", "destination=", "inputFileList=", "outputFileList=", \
462	+	"protocol=","option=", "middleware=", "srm_version=", "debug", "help"]
463	+	try:
464	+	opts, args = getopt.getopt( sys.argv[1:], "", allowedOpt )
465	+	except getopt.GetoptError, err:
466	+	print err
467	+	HelpOptions()
468	+	sys.exit(2)
469	+
470	+	dictArgs = HelpOptions(opts)
471		try:
472	<	cmscp_ = cmscp(sys.argv[1:])
472	>	cmscp_ = cmscp(dictArgs)
473		cmscp_.run()
474	<	except:
475	<	pass
474	>	except Exception, ex :
475	>	print str(ex)
476

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines