ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/cmscp.py

Comparing COMP/CRAB/python/cmscp.py (file contents):
Revision 1.2 by spiga, Mon Sep 22 09:03:17 2008 UTC vs.
Revision 1.91 by fanzago, Mon Aug 29 09:48:04 2011 UTC

#	Line 1 | Line 1
1		#!/usr/bin/env python
2	<
3	<	import sys, getopt, string
4	<	import os, popen2
5	<	from ProdCommon.Storage.SEAPI.SElement import SElement, FullPath
2	>	import sys, os
3	>	import time, random
4	>	try:
5	>	import json
6	>	except:
7	>	import simplejson as json
8	>	from ProdCommon.Storage.SEAPI.SElement import SElement, FullPath
9		from ProdCommon.Storage.SEAPI.SBinterface import *
10	<
10	>	from ProdCommon.Storage.SEAPI.Exceptions import *
11
12
13		class cmscp:
14	<	def __init__(self, argv):
14	>	def __init__(self, args):
15		"""
16		cmscp
17	<
15	<	safe copy of local file in current directory to remote SE via lcg_cp/srmcp,
17	>	safe copy of local file  to/from remote SE via lcg_cp/srmcp,
18		including success checking  version also for CAF using rfcp command to copy the output to SE
19		input:
20		$1 middleware (CAF, LSF, LCG, OSG)
21		$2 local file (the absolute path of output file or just the name if it's in top dir)
22		$3 if needed: file name (the output file name)
23		$5 remote SE (complete endpoint)
24	<	$6 srm version
24	>	$6 srm version
25	>	--for_lfn $LFNBaseName
26		output:
27		return 0 if all ok
28		return 60307 if srmcp failed
29		return 60303 if file already exists in the SE
30		"""
31	<	#set default
31	>	self.params = {"source":'', "destination":'','destinationDir':'', "inputFileList":'', "outputFileList":'', \
32	>	"protocol":'', "option":'', "middleware":'', "srm_version":'srmv2', "for_lfn":'', "se_name":'', "surl_for_grid":''}
33		self.debug = 0
34	<	self.source = ''
35	<	self.destination = ''
36	<	self.file_to_copy = []
37	<	self.remote_file_name = []
38	<	self.protocol = ''
39	<	self.middleware = ''
40	<	self.srmv = ''
41	<
42	<	try:
43	<	opts, args = getopt.getopt(argv, "", ["source=", "destination=", "inputFileList=", "outputFileList=", \
44	<	"protocol=", "middleware=", "srm_version=", "debug", "help"])
45	<	except getopt.GetoptError:
46	<	print self.usage()
43	<	sys.exit(2)
44	<
45	<	self.setAndCheck(opts)
46	<
34	>	#### for fallback copy
35	>	self.local_stage = 0
36	>	self.params.update( args )
37	>	## timeout needed for subprocess command of SEAPI
38	>	## they should be a bit higher then the corresponding passed by command line
39	>	## default values
40	>	self.subprocesstimeout = { \
41	>	'copy':   3600, \
42	>	'exists': 1200, \
43	>	'delete': 1200, \
44	>	'size':   1200 \
45	>	}
46	>
47		return
48
49	<	def setAndCheck( self, opts ):
49	>	def processOptions( self ):
50		"""
51	<	Set and check command line parameter
51	>	check command line parameter
52		"""
53	<	if not opts :
54	<	print self.usage()
55	<	sys.exit()
56	<	for opt, arg in opts :
57	<	if opt  == "--help" :
58	<	print self.usage()
59	<	sys.exit()
60	<	elif opt == "--debug" :
61	<	self.debug = 1
62	<	elif opt == "--source" :
63	<	self.source = arg
64	<	elif opt == "--destination":
65	<	self.destination = arg
66	<	elif opt == "--inputFileList":
67	<	infile = arg
68	<	elif opt == "--outputFileList":
69	<	out_file
70	<	elif opt == "--protocol":
71	<	self.protocol = arg
72	<	elif opt == "--middleware":
73	<	self.middleware = arg
74	<	elif opt == "--srm_version":
75	<	self.srmv = arg
76	<
77	<	# source and dest cannot be undefined at same time
78	<	if self.source == '' and self.destination == '':
79	<	print self.usage()
80	<	sys.exit()
81	<	# if middleware is not defined --> protocol cannot be empty
82	<	if self.middleware == '' and self.protocol == '':
83	<	print self.usage()
84	<	sys.exit()
85	<	# input file must be defined
86	<	if infile == '':
87	<	print self.usage()
88	<	sys.exit()
53	>	if 'help' in self.params.keys(): HelpOptions()
54	>	if 'debug' in self.params.keys(): self.debug = 1
55	>	if 'local_stage' in self.params.keys(): self.local_stage = 1
56	>
57	>	# source and dest cannot be undefined at same time
58	>	if not self.params['source']  and not self.params['destination'] :
59	>	HelpOptions()
60	>	# if middleware is not defined --> protocol cannot be empty
61	>	if not self.params['middleware'] and not self.params['protocol'] :
62	>	HelpOptions()
63	>
64	>	# input file must be defined
65	>	if not self.params['inputFileList'] :
66	>	HelpOptions()
67		else:
68	<	if infile.find(','):
69	<	[self.file_to_copy.append(x.strip()) for x in infile.split(',')]
70	<	else:
71	<	self.file_to_copy.append(infile)
72	<
73	<	## TO DO:
96	<	#### add check for outFiles
97	<	#### add map {'inFileNAME':'outFileNAME'} to change out name
68	>	file_to_copy=[]
69	>	if self.params['inputFileList'].find(','):
70	>	[file_to_copy.append(x.strip()) for x in self.params['inputFileList'].split(',')]
71	>	else:
72	>	file_to_copy.append(self.params['inputFileList'])
73	>	self.params['inputFileList'] = file_to_copy
74
75	<	return
75	>	if not self.params['for_lfn'] and self.local_stage == 1 : HelpOptions()
76	>
77
78	<	def run( self ):
102	<	"""
103	<	Check if running on UI (no $middleware) or
104	<	on WN (on the Grid), and take different action
78	>	def run( self ):
79		"""
80	<	if self.middleware :
81	<	results = self.stager()
80	>	Check if running on UI (no $middleware) or
81	>	on WN (on the Grid), and take different action
82	>	"""
83	>	self.processOptions()
84	>	if self.debug: print 'calling run() : \n'
85	>	# stage out from WN
86	>	if self.params['middleware'] :
87	>	results = self.stager(self.params['middleware'],self.params['inputFileList'])
88	>	self.writeJsonFile(results)
89	>	self.finalReport(results)
90	>	# Local interaction with SE
91		else:
92	<	results = self.copy( self.file_to_copy, self.protocol )
92	>	results = self.copy(self.params['inputFileList'], self.params['protocol'], self.params['option'] )
93	>	self.writeJsonFile(results)
94	>	return results
95	>
96	>	def writeJsonFile( self, results ):
97	>	"""
98	>	write a json file containing copy results for each file
99	>	"""
100	>	if self.debug:
101	>	print 'in writeJsonFile() : \n'
102	>	print "---->>>> in writeJsonFile results =  ", results
103	>	jsonOut = "resultCopyFile"
104	>	if os.getenv("RUNTIME_AREA"):
105	>	jsonOut = "%s/resultCopyFile"%os.getenv("RUNTIME_AREA")
106	>	fp = open(jsonOut, 'w')
107	>	json.dump(results, fp)
108	>	fp.close()
109	>	if self.debug:
110	>	print '    reading resultCopyFile : \n'
111	>	lp = open(jsonOut, "r")
112	>	inputDict = json.load(lp)
113	>	lp.close()
114	>	print "    inputDict = ", inputDict
115	>	return
116
117	<	self.finalReport(results,self.middleware)
117	>	def checkLcgUtils( self ):
118	>	"""
119	>	_checkLcgUtils_
120	>	check the lcg-utils version and report
121	>	"""
122	>	import commands
123	>	cmd = "lcg-cp --version | grep lcg_util"
124	>	status, output = commands.getstatusoutput( cmd )
125	>	num_ver = -1
126	>	if output.find("not found") == -1 or status == 0:
127	>	temp = output.split("-")
128	>	version = ""
129	>	if len(temp) >= 2:
130	>	version = output.split("-")[1]
131	>	temp = version.split(".")
132	>	if len(temp) >= 1:
133	>	num_ver = int(temp[0])*10
134	>	num_ver += int(temp[1])
135	>	return num_ver
136
137	<	return
114	<
115	<	def setProtocol( self ):
137	>	def setProtocol( self, middleware ):
138		"""
139		define the allowed potocols based on $middlware
140	<	which depend on scheduler
140	>	which depend on scheduler
141		"""
142	<	if self.middleware.lower() in ['osg','lcg']:
143	<	supported_protocol = ['srm-lcg','srmv2']
144	<	elif self.middleware.lower() in ['lsf','caf']:
145	<	supported_protocol = ['rfio']
142	>	# default To be used with "middleware"
143	>	if self.debug:
144	>	print 'setProtocol() :\n'
145	>	print '\tmiddleware =  %s utils \n'%middleware
146	>
147	>	lcgOpt={'srmv1':'-b -D srmv1  -t 2400 --verbose',
148	>	'srmv2':'-b -D srmv2  -t 2400 --verbose'}
149	>	if self.checkLcgUtils() >= 17:
150	>	lcgOpt={'srmv1':'-b -D srmv1 --srm-timeout 2400 --sendreceive-timeout 2400 --connect-timeout 300 --verbose',
151	>	'srmv2':'-b -D srmv2 --srm-timeout 2400 --sendreceive-timeout 2400 --connect-timeout 300 --verbose'}
152	>
153	>	srmOpt={'srmv1':' -report ./srmcp.report -retry_timeout 480000 -retry_num 3 -streams_num=1 ',
154	>	'srmv2':' -report=./srmcp.report -retry_timeout=480000 -retry_num=3 -storagetype=permanent '}
155	>	rfioOpt=''
156	>	#### FEDE FOR XROOTD #########
157	>	xrootdOpt=''
158	>	#############################
159	>
160	>	supported_protocol = None
161	>	if middleware.lower() in ['osg','lcg','condor','sge']:
162	>	supported_protocol = [('srm-lcg',lcgOpt[self.params['srm_version']])]#,\
163	>	#  (self.params['srm_version'],srmOpt[self.params['srm_version']])]
164	>	#elif middleware.lower() in ['lsf','caf']:
165	>	elif middleware.lower() in ['lsf']:
166	>	supported_protocol = [('rfio',rfioOpt)]
167	>	elif middleware.lower() in ['pbs']:
168	>	supported_protocol = [('rfio',rfioOpt),('local','')]
169	>	elif middleware.lower() in ['arc']:
170	>	supported_protocol = [('srmv2','-debug'),('srmv1','-debug')]
171	>	#### FEDE FOR XROOTD ##########
172	>	elif middleware.lower() in ['caf']:
173	>	if self.params['protocol']:
174	>	supported_protocol = [(self.params['protocol'], '')]
175	>	else:
176	>	supported_protocol = [('rfio',rfioOpt)]
177	>	#######################################
178		else:
179	<	## here we can add support for any kind of protocol,
179	>	## here we can add support for any kind of protocol,
180		## maybe some local schedulers need something dedicated
181		pass
182		return supported_protocol
183
184	<	def stager( self ):
184	>
185	>	def checkCopy (self, copy_results, len_list_files, prot):
186	>	"""
187	>	Checks the status of copy and update result dictionary
188	>	"""
189	>
190	>	list_retry = []
191	>	list_not_existing = []
192	>	list_already_existing = []
193	>	list_fallback = []
194	>	list_ok = []
195	>
196	>	if self.debug:
197	>	print 'in checkCopy() :\n'
198	>
199	>	for file, dict in copy_results.iteritems():
200	>	er_code = dict['erCode']
201	>	if er_code == '0':
202	>	list_ok.append(file)
203	>	reason = 'Copy succedeed with %s utils'%prot
204	>	dict['reason'] = reason
205	>	elif er_code == '60308':
206	>	list_fallback.append( file )
207	>	reason = 'Copy succedeed with %s utils'%prot
208	>	dict['reason'] = reason
209	>	elif er_code == '60302':
210	>	list_not_existing.append( file )
211	>	elif er_code == '60303':
212	>	list_already_existing.append( file )
213	>	else :
214	>	list_retry.append( file )
215	>
216	>	if self.debug:
217	>	print "\t file %s \n"%file
218	>	print "\t dict['erCode'] %s \n"%dict['erCode']
219	>	print "\t dict['reason'] %s \n"%dict['reason']
220	>
221	>	upDict = self.updateReport(file, er_code, dict['reason'])
222	>
223	>	copy_results.update(upDict)
224	>
225	>	msg = ''
226	>	if len(list_ok) != 0:
227	>	msg += '\tCopy of %s succedeed with %s utils\n'%(str(list_ok),prot)
228	>	if len(list_ok) != len_list_files :
229	>	if len(list_fallback)!=0:
230	>	msg += '\tCopy of %s succedeed with %s utils in the fallback SE\n'%(str(list_fallback),prot)
231	>	if len(list_retry)!=0:
232	>	msg += '\tCopy of %s failed using %s for files \n'%(str(list_retry),prot)
233	>	if len(list_not_existing)!=0:
234	>	msg += '\tCopy of %s failed using %s : files not found \n'%(str(list_not_existing),prot)
235	>	if len(list_already_existing)!=0:
236	>	msg += '\tCopy of %s failed using %s : files already existing\n'%(str(list_already_existing),prot)
237	>	if self.debug : print msg
238	>	return copy_results, list_ok, list_retry, list_fallback
239	>
240	>	def check_for_retry_localSE (self, copy_results):
241	>	"""
242	>	Checks the status of copy and create the list of file to copy to CloseSE
243	>	"""
244	>	list_retry_localSE = []
245	>
246	>	if self.debug:
247	>	print 'in check_for_retry_localSE() :\n'
248	>	print "\t results in check local = ", copy_results
249	>	for file, dict in copy_results.iteritems():
250	>	er_code = dict['erCode']
251	>	if er_code != '0' and  er_code != '60302' and er_code != '60308':
252	>	list_retry_localSE.append( file )
253	>
254	>	if self.debug:
255	>	print "\t file %s \n"%file
256	>	print "\t dict['erCode'] %s \n"%dict['erCode']
257	>	print "\t dict['reason'] %s \n"%dict['reason']
258	>
259	>	return list_retry_localSE
260	>
261	>
262	>	def LocalCopy(self, list_retry, results):
263	>	"""
264	>	Tries the stage out to the CloseSE
265	>	"""
266	>	if self.debug:
267	>	print 'in LocalCopy() :\n'
268	>	print '\t list_retry %s utils \n'%list_retry
269	>	print '\t len(list_retry) %s \n'%len(list_retry)
270	>
271	>	list_files = list_retry
272	>	self.params['inputFileList']=list_files
273	>
274	>	### copy backup
275	>	from ProdCommon.FwkJobRep.SiteLocalConfig import loadSiteLocalConfig
276	>	siteCfg = loadSiteLocalConfig()
277	>	catalog = siteCfg.localStageOut.get("catalog", None)
278	>	tfc = siteCfg.trivialFileCatalog()
279	>	seName = siteCfg.localStageOutSEName()
280	>	if seName is None:
281	>	seName = ""
282	>	option = siteCfg.localStageOutOption()
283	>	if option is None:
284	>	option = ""
285	>	implName = siteCfg.localStageOutCommand()
286	>	if implName is None:
287	>	implName = ""
288	>
289	>	if (implName == 'srm'):
290	>	protocol = 'srmv2'
291	>	elif (implName == 'srmv2-lcg'):
292	>	protocol = 'srm-lcg'
293	>	option = option + ' -b -D srmv2 '
294	>	elif (implName == 'rfcp-CERN'):
295	>	protocol = 'rfio'
296	>	elif (implName == 'rfcp'):
297	>	protocol = 'rfio'
298	>	elif (implName == 'cp'):
299	>	protocol = 'local'
300	>	else: protocol = implName
301	>
302	>	self.params['protocol']=protocol
303	>	self.params['option']=option
304	>
305	>	if self.debug:
306	>	print '\t siteCFG %s \n'%siteCfg
307	>	print '\t catalog %s \n'%catalog
308	>	print '\t tfc %s '%tfc
309	>	print '\t fallback seName %s \n'%seName
310	>	print "\t fallback protocol %s \n"%protocol
311	>	print "\t fallback option %s \n"%option
312	>	print "\t self.params['inputFileList'] %s \n"%self.params['inputFileList']
313	>
314	>	if (str(self.params['for_lfn']).find("/store/") == 0):
315	>	temp = str(self.params['for_lfn']).replace("/store/","/store/temp/",1)
316	>	self.params['for_lfn']= temp
317	>
318	>	if ( self.params['for_lfn'][-1] != '/' ) : self.params['for_lfn'] = self.params['for_lfn'] + '/'
319	>
320	>	file_backup=[]
321	>	file_backup_surlgrid=[]
322	>	for input in self.params['inputFileList']:
323	>	file = self.params['for_lfn'] + os.path.basename(input)
324	>	surl = tfc.matchLFN(tfc.preferredProtocol, file)
325	>
326	>	###### FEDE TEST_FOR_SURL_GRID
327	>	surl_for_grid = tfc.matchLFN('srmv2', file)
328	>	if (surl_for_grid == None):
329	>	surl_for_grid = tfc.matchLFN('srmv2-lcg', file)
330	>	if (surl_for_grid == None):
331	>	surl_for_grid = tfc.matchLFN('srm', file)
332	>	if surl_for_grid:
333	>	file_backup_surlgrid.append(surl_for_grid)
334	>	######
335	>
336	>	file_backup.append(surl)
337	>	if self.debug:
338	>	print '\t for_lfn %s \n'%self.params['for_lfn']
339	>	print '\t file %s \n'%file
340	>	print '\t surl %s \n'%surl
341	>
342	>	destination=os.path.dirname(file_backup[0])
343	>	if ( destination[-1] != '/' ) : destination = destination + '/'
344	>	self.params['destination']=destination
345	>
346	>	self.params['se_name']=seName
347	>
348	>	######
349	>	if (len(file_backup_surlgrid)>0) :
350	>	surl_for_grid=os.path.dirname(file_backup_surlgrid[0])
351	>	if ( surl_for_grid[-1] != '/' ) : surl_for_grid = surl_for_grid + '/'
352	>	else:
353	>	surl_for_grid=''
354	>
355	>	print "surl_for_grid = ", surl_for_grid
356	>	self.params['surl_for_grid']=surl_for_grid
357	>	#####
358	>
359	>	if self.debug:
360	>	print '\tIn LocalCopy trying the stage out with: \n'
361	>	print "\tself.params['destination'] %s \n"%self.params['destination']
362	>	print "\tself.params['protocol'] %s \n"%self.params['protocol']
363	>	print "\tself.params['option'] %s \n"%self.params['option']
364	>
365	>	localCopy_results = self.copy( self.params['inputFileList'], self.params['protocol'], self.params['option'], backup='yes' )
366	>
367	>	if localCopy_results.keys() == [''] or localCopy_results.keys() == '' :
368	>	results.update(localCopy_results)
369	>	else:
370	>	localCopy_results, list_ok, list_retry, list_fallback = self.checkCopy(localCopy_results, len(list_files), self.params['protocol'])
371	>
372	>	results.update(localCopy_results)
373	>	if self.debug:
374	>	print "\t localCopy_results = %s \n"%localCopy_results
375	>	return results
376	>
377	>	def stager( self, middleware, list_files ):
378		"""
379		Implement the logic for remote stage out
380		"""
134	–	protocols = self.setProtocol()
135	–	count=0
136	–	list_files = self.file_to_copy
137	–	results={}
138	–	for prot in protocols:
139	–	if self.debug: print 'Trying stage out with %s utils \n'%prot
140	–	copy_results = self.copy( list_files, prot )
141	–	list_retry = []
142	–	list_existing = []
143	–	list_ok = []
144	–	for file, dict in copy_results.iteritems():
145	–	er_code = dict['erCode']
146	–	if er_code == '60307': list_retry.append( file )
147	–	elif er_code == '60303': list_existing.append( file )
148	–	else:
149	–	list_ok.append(file)
150	–	reason = 'Copy succedeed with %s utils'%prot
151	–	upDict = self.updateReport(file, er_code, reason)
152	–	copy_results.update(upDict)
153	–	results.update(copy_results)
154	–	if len(list_ok) != 0:
155	–	msg = 'Copy of %s succedeed with %s utils\n'%(str(list_ok),prot)
156	–	# print msg
157	–	if len(list_ok) == len(list_files) :
158	–	break
159	–	else:
160	–	#       print 'Copy of files %s failed using %s...\n'%(str(list_retry)+str(list_existing),prot)
161	–	if len(list_retry): list_files = list_retry
162	–	else: break
163	–	count =+1
381
382	<	#### TODO Daniele
383	<	#check is something fails and created related dict
384	<	#      backup = self.analyzeResults(results)
385	<
386	<	#      if backup :
387	<	#          msg = 'WARNING: backup logic is under implementation\n'
388	<	#          #backupDict = self.backup()
389	<	#          ### NOTE: IT MUST RETURN a DICT contains also LFN and SE Name
390	<	#          results.update(backupDict)
391	<	#          print msg
382	>	if self.debug:
383	>	print 'stager() :\n'
384	>	print '\tmiddleware %s\n'%middleware
385	>	print '\tlist_files %s\n'%list_files
386	>
387	>	results={}
388	>	for prot, opt in self.setProtocol( middleware ):
389	>	if self.debug:
390	>	print '\tTrying the stage out with %s utils \n'%prot
391	>	print '\tand options %s\n'%opt
392	>
393	>	copy_results = self.copy( list_files, prot, opt )
394	>	if copy_results.keys() == [''] or copy_results.keys() == '' :
395	>	results.update(copy_results)
396	>	else:
397	>	copy_results, list_ok, list_retry, list_fallback = self.checkCopy(copy_results, len(list_files), prot)
398	>	results.update(copy_results)
399	>	if len(list_ok) == len(list_files) :
400	>	break
401	>	if len(list_retry):
402	>	list_files = list_retry
403	>	#### FEDE added ramdom time before the retry copy with other protocol
404	>	sec =  240 * random.random()
405	>	sec = sec + 60
406	>	time.sleep(sec)
407	>	else: break
408	>
409	>	if self.local_stage:
410	>	list_retry_localSE = self.check_for_retry_localSE(results)
411	>	if len(list_retry_localSE):
412	>	if self.debug:
413	>	print "\t list_retry_localSE %s \n"%list_retry_localSE
414	>	results = self.LocalCopy(list_retry_localSE, results)
415	>
416	>	if self.debug:
417	>	print "\t results %s \n"%results
418		return results
419
420		def initializeApi(self, protocol ):
421		"""
422	<	Instantiate storage interface
422	>	Instantiate storage interface
423		"""
424	<	source_prot = protocol
425	<	dest_prot = protocol
426	<	if self.source == '' : source_prot = 'local'
427	<	Source_SE  = self.storageInterface( self.source, source_prot )
428	<	if self.destination == '' : dest_prot = 'local'
429	<	Destination_SE = self.storageInterface( self.destination, dest_prot )
424	>	if self.debug : print 'initializeApi() :\n'
425	>	self.source_prot = protocol
426	>	self.dest_prot = protocol
427	>	if not self.params['source'] : self.source_prot = 'local'
428	>	Source_SE  = self.storageInterface( self.params['source'], self.source_prot )
429	>	if not self.params['destination'] :
430	>	self.dest_prot = 'local'
431	>	Destination_SE = self.storageInterface( self.params['destinationDir'], self.dest_prot )
432	>	else:
433	>	Destination_SE = self.storageInterface( self.params['destination'], self.dest_prot )
434
435		if self.debug :
436	<	print '(source=%s,  protocol=%s)'%(self.source, source_prot)
437	<	print '(destination=%s,  protocol=%s)'%(self.destination, dest_prot)
436	>	msg  = '\t(source=%s,  protocol=%s)'%(self.params['source'], self.source_prot)
437	>	msg += '\t(destination=%s,  protocol=%s)'%(self.params['destination'], self.dest_prot)
438	>	msg += '\t(destinationDir=%s,  protocol=%s)'%(self.params['destinationDir'], self.dest_prot)
439	>	print msg
440
441		return Source_SE, Destination_SE
442
443	<	def copy( self, list_file, protocol ):
443	>	def copy( self, list_file, protocol, options, backup='no' ):
444		"""
445	<	Make the real file copy using SE API
445	>	Make the real file copy using SE API
446		"""
447	+	msg = ""
448	+	results = {}
449		if self.debug :
450	<	print 'copy(): using %s protocol'%protocol
451	<	Source_SE, Destination_SE = self.initializeApi( protocol )
452	<
453	<	# create remote dir
454	<	if protocol in ['gridftp','rfio']:
455	<	self.createDir( Destination_SE, protocol )
456	<
450	>	msg  = 'copy() :\n'
451	>	msg += '\tusing %s protocol\n'%protocol
452	>	msg += '\tusing %s options\n'%options
453	>	msg += '\tlist_file %s\n'%list_file
454	>	print msg
455	>	try:
456	>	Source_SE, Destination_SE = self.initializeApi( protocol )
457	>	except Exception, ex:
458	>	for filetocopy in list_file:
459	>	results.update( self.updateReport(filetocopy, '-1', str(ex)))
460	>	return results
461	>
462	>	self.hostname = Destination_SE.hostname
463	>	if Destination_SE.protocol in ['gridftp','rfio','srmv2','hadoop','local']:
464	>	try:
465	>	self.createDir( Destination_SE, Destination_SE.protocol )
466	>	except OperationException, ex:
467	>	for filetocopy in list_file:
468	>	results.update( self.updateReport(filetocopy, '60316', str(ex)))
469	>	return results
470	>	## when the client commands are not found (wrong env or really missing)
471	>	except MissingCommand, ex:
472	>	msg = "ERROR %s %s" %(str(ex), str(ex.detail))
473	>	for filetocopy in list_file:
474	>	results.update( self.updateReport(filetocopy, '10041', msg))
475	>	return results
476	>	except Exception, ex:
477	>	msg = "ERROR %s" %(str(ex))
478	>	for filetocopy in list_file:
479	>	results.update( self.updateReport(filetocopy, '-1', msg))
480	>	return results
481		## prepare for real copy  ##
482	<	sbi = SBinterface( Source_SE, Destination_SE )
483	<	sbi_dest = SBinterface(Destination_SE)
484	<
485	<	results = {}
486	<	## loop over the complete list of files
487	<	for filetocopy in list_file:
488	<	if self.debug : print 'start real copy for %s'%filetocopy
489	<	ErCode, msg = self.checkFileExist( sbi_dest, os.path.basename(filetocopy) )
490	<	if ErCode == '0':
491	<	ErCode, msg = self.makeCopy( sbi, filetocopy )
492	<	if self.debug : print 'Copy results for %s is %s'%( os.path.basename(filetocopy) ,ErCode)
482	>	try :
483	>	sbi = SBinterface( Source_SE, Destination_SE )
484	>	sbi_dest = SBinterface(Destination_SE)
485	>	sbi_source = SBinterface(Source_SE)
486	>	except ProtocolMismatch, ex:
487	>	msg  = "ERROR : Unable to create SBinterface with %s protocol"%protocol
488	>	msg += str(ex)
489	>	for filetocopy in list_file:
490	>	results.update( self.updateReport(filetocopy, '-1', msg))
491	>	return results
492	>
493	>
494	>	## loop over the complete list of files
495	>	for filetocopy in list_file:
496	>	if self.debug : print '\tStart real copy for %s'%filetocopy
497	>	try :
498	>	ErCode, msg = self.checkFileExist( sbi_source, sbi_dest, filetocopy, options )
499	>	except Exception, ex:
500	>	ErCode = '60307'
501	>	msg = str(ex)
502	>	if ErCode == '0':
503	>	ErCode, msg = self.makeCopy( sbi, filetocopy , options, protocol,sbi_dest )
504	>	if (ErCode == '0') and (backup == 'yes'):
505	>	ErCode = '60308'
506	>	if self.debug : print '\tCopy results for %s is %s'%( os.path.basename(filetocopy), ErCode)
507		results.update( self.updateReport(filetocopy, ErCode, msg))
508		return results
220	–
221	–	def updateReport(self, file, erCode, reason, lfn='', se='' ):
222	–	"""
223	–	Update the final stage out infos
224	–	"""
225	–	jobStageInfo={}
226	–	jobStageInfo['erCode']=erCode
227	–	jobStageInfo['reason']=reason
228	–	jobStageInfo['lfn']=lfn
229	–	jobStageInfo['se']=se
509
231	–	report = { file : jobStageInfo}
232	–	return report
233	–
234	–	def finalReport( self , results, middleware ):
235	–	"""
236	–	It should return a clear list of LFNs for each SE where data are stored.
237	–	allow "crab -copyLocal" or better "crab -copyOutput". TO_DO.
238	–	"""
239	–	if middleware:
240	–	outFile = open('cmscpReport.sh',"a")
241	–	cmscp_exit_status = 0
242	–	txt = ''
243	–	for file, dict in results.iteritems():
244	–	if dict['lfn']=='':
245	–	lfn = '$LFNBaseName/'+os.path.basename(file)
246	–	se  = '$SE'
247	–	else:
248	–	lfn = dict['lfn']+os.pat.basename(file)
249	–	se = dict['se']
250	–	#dict['lfn'] # to be implemented
251	–	txt +=  'echo "Report for File: '+file+'"\n'
252	–	txt +=  'echo "LFN: '+lfn+'"\n'
253	–	txt +=  'echo "StorageElement: '+se+'"\n'
254	–	txt += 'echo "StageOutExitStatusReason ='+dict['reason']+'" | tee -a $RUNTIME_AREA/$repo\n'
255	–	txt += 'echo "StageOutSE = '+se+'" >> $RUNTIME_AREA/$repo\n'
256	–	if dict['erCode'] != '0':
257	–	cmscp_exit_status = dict['erCode']
258	–	txt += '\n'
259	–	txt += 'export StageOutExitStatus='+str(cmscp_exit_status)+'\n'
260	–	txt +=  'echo "StageOutExitStatus = '+str(cmscp_exit_status)+'" | tee -a $RUNTIME_AREA/$repo\n'
261	–	outFile.write(str(txt))
262	–	outFile.close()
263	–	else:
264	–	for file, code in results.iteritems():
265	–	print 'error code = %s for file %s'%(code,file)
266	–	return
510
511		def storageInterface( self, endpoint, protocol ):
512		"""
513	<	Create the storage interface.
513	>	Create the storage interface.
514		"""
515	+	if self.debug : print 'storageInterface():\n'
516		try:
517		interface = SElement( FullPath(endpoint), protocol )
518	<	except Exception, ex:
519	<	msg = ''
520	<	if self.debug : msg = str(ex)+'\n'
521	<	msg += "ERROR : Unable to create interface with %s protocol\n"%protocol
278	<	print msg
518	>	except ProtocolUnknown, ex:
519	>	msg  = "ERROR : Unable to create interface with %s protocol"%protocol
520	>	msg += str(ex)
521	>	raise Exception(msg)
522
523		return interface
524
282	–	def checkDir(self, Destination_SE, protocol):
283	–	'''
284	–	ToBeImplemented NEEDED for castor
285	–	'''
286	–	return
287	–
525		def createDir(self, Destination_SE, protocol):
526		"""
527	<	Create remote dir for gsiftp/rfio REALLY TEMPORARY
528	<	this should be transparent at SE API level.
527	>	Create remote dir for gsiftp REALLY TEMPORARY
528	>	this should be transparent at SE API level.
529		"""
530	<	ErCode = '0'
531	<	msg_1 = ''
530	>	if self.debug : print 'createDir():\n'
531	>	msg = ''
532		try:
533		action = SBinterface( Destination_SE )
534		action.createDir()
535	<	if self.debug: print "The directory has been created using protocol %s\n"%protocol
536	<	except Exception, ex:
537	<	msg = ''
538	<	if self.debug : msg = str(ex)+'\n'
539	<	msg_1 = "ERROR: problem with the directory creation using %s protocol \n"%protocol
540	<	msg += msg_1
541	<	ErCode = '60316'
542	<	#print msg
543	<
544	<	return ErCode, msg_1
535	>	if self.debug: print "\tThe directory has been created using protocol %s"%protocol
536	>	except TransferException, ex:
537	>	msg  = "ERROR: problem with the directory creation using %s protocol "%protocol
538	>	msg += str(ex)
539	>	if self.debug :
540	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
541	>	dbgmsg += '\t'+str(ex.output)+'\n'
542	>	print dbgmsg
543	>	raise Exception(msg)
544	>	except OperationException, ex:
545	>	msg  = "ERROR: problem with the directory creation using %s protocol "%protocol
546	>	msg += str(ex)
547	>	if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n'
548	>	raise Exception(msg)
549	>	except MissingDestination, ex:
550	>	msg  = "ERROR: problem with the directory creation using %s protocol "%protocol
551	>	msg += str(ex)
552	>	if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n'
553	>	raise Exception(msg)
554	>	except AlreadyExistsException, ex:
555	>	if self.debug: print "\tThe directory already exist"
556	>	pass
557	>	except Exception, ex:
558	>	msg = "ERROR %s %s" %(str(ex), str(ex.detail))
559	>	if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n'
560	>	raise Exception(msg)
561	>	return msg
562
563	<	def checkFileExist(self, sbi, filetocopy):
563	>	def checkFileExist( self, sbi_source, sbi_dest, filetocopy, option ):
564		"""
565	<	Check if file to copy already exist
566	<	"""
567	<	try:
568	<	check = sbi.checkExists(filetocopy)
315	<	except Exception, ex:
316	<	msg = ''
317	<	if self.debug : msg = str(ex)+'\n'
318	<	msg += "ERROR: problem with check File Exist using %s protocol \n"%protocol
319	<	# print msg
565	>	Check both if source file exist AND
566	>	if destination file ALREADY exist.
567	>	"""
568	>	if self.debug : print 'checkFileExist():\n'
569		ErCode = '0'
570		msg = ''
571	<	if check :
572	<	ErCode = '60303'
573	<	msg = "file %s already exist"%filetocopy
574	<	print msg
571	>	f_tocopy=filetocopy
572	>	if self.source_prot != 'local':f_tocopy = os.path.basename(filetocopy)
573	>	try:
574	>	checkSource = sbi_source.checkExists( f_tocopy , opt=option, tout = self.subprocesstimeout['exists'] )
575	>	if self.debug : print '\tCheck for local file %s existance executed \n'%f_tocopy
576	>	except OperationException, ex:
577	>	msg  ='ERROR: problems checking source file %s existance'%filetocopy
578	>	msg += str(ex)
579	>	if self.debug :
580	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
581	>	dbgmsg += '\t'+str(ex.output)+'\n'
582	>	print dbgmsg
583	>	raise Exception(msg)
584	>	except WrongOption, ex:
585	>	msg  ='ERROR: problems checking source file %s existance'%filetocopy
586	>	msg += str(ex)
587	>	if self.debug :
588	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
589	>	dbgmsg += '\t'+str(ex.output)+'\n'
590	>	print dbgmsg
591	>	raise Exception(msg)
592	>	except MissingDestination, ex:
593	>	msg  ='ERROR: problems checking source file %s existance'%filetocopy
594	>	msg += str(ex)
595	>	if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n'
596	>	raise Exception(msg)
597	>	## when the client commands are not found (wrong env or really missing)
598	>	except MissingCommand, ex:
599	>	ErCode = '10041'
600	>	msg = "ERROR %s %s" %(str(ex), str(ex.detail))
601	>	return ErCode, msg
602	>	if not checkSource :
603	>	ErCode = '60302'
604	>	msg = "ERROR file %s does not exist"%os.path.basename(filetocopy)
605	>	return ErCode, msg
606	>	f_tocopy=os.path.basename(filetocopy)
607	>	try:
608	>	check = sbi_dest.checkExists( f_tocopy, opt=option, tout = self.subprocesstimeout['exists'] )
609	>	if self.debug : print '\tCheck for remote file %s existance executed \n'%f_tocopy
610	>	if self.debug : print '\twith exit code = %s \n'%check
611	>	except OperationException, ex:
612	>	msg  = 'ERROR: problems checking if file %s already exist'%filetocopy
613	>	msg += str(ex)
614	>	if self.debug :
615	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
616	>	dbgmsg += '\t'+str(ex.output)+'\n'
617	>	print dbgmsg
618	>	raise Exception(msg)
619	>	except WrongOption, ex:
620	>	msg  = 'ERROR problems checking if file % already exists'%filetocopy
621	>	msg += str(ex)
622	>	if self.debug :
623	>	msg += '\t'+msg+'\n\t'+str(ex.detail)+'\n'
624	>	msg += '\t'+str(ex.output)+'\n'
625	>	raise Exception(msg)
626	>	except MissingDestination, ex:
627	>	msg  ='ERROR problems checking if destination file % exists'%filetocopy
628	>	msg += str(ex)
629	>	if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n'
630	>	raise Exception(msg)
631	>	## when the client commands are not found (wrong env or really missing)
632	>	except MissingCommand, ex:
633	>	ErCode = '10041'
634	>	msg = "ERROR %s %s" %(str(ex), str(ex.detail))
635	>	return ErCode, msg
636	>	if check :
637	>	ErCode = '60303'
638	>	msg = "file %s already exist"%os.path.basename(filetocopy)
639
640	<	return ErCode,msg
640	>	return ErCode, msg
641
642	<	def makeCopy(self, sbi, filetocopy ):
642	>	def makeCopy(self, sbi, filetocopy, option, protocol, sbi_dest ):
643		"""
644	<	call the copy API.
644	>	call the copy API.
645		"""
646	<	path = os.path.dirname(filetocopy)
646	>	if self.debug : print 'makeCopy():\n'
647	>	path = os.path.dirname(filetocopy)
648		file_name =  os.path.basename(filetocopy)
649		source_file = filetocopy
650		dest_file = file_name ## to be improved supporting changing file name  TODO
651	<	if self.source == '' and path == '':
651	>	if self.params['source'] == '' and path == '':
652		source_file = os.path.abspath(filetocopy)
653	<	elif self.destination =='':
654	<	dest_file = os.path.join(os.getcwd(),file_name)
655	<	elif self.source != '' and self.destination != '' :
656	<	source_file = file_name
653	>	elif self.params['destination'] =='':
654	>	destDir = self.params.get('destinationDir',os.getcwd())
655	>	dest_file = os.path.join(destDir,file_name)
656	>	elif self.params['source'] != '' and self.params['destination'] != '' :
657	>	source_file = file_name
658	>
659		ErCode = '0'
660		msg = ''
661	+
662	+	copy_option = option
663	+	if  self.params['option'].find('space_token')>=0:
664	+	space_token=self.params['option'].split('=')[1]
665	+	if protocol == 'srmv2': copy_option = '%s -space_token=%s'%(option,space_token)
666	+	if protocol == 'srm-lcg': copy_option = '%s -S %s'%(option,space_token)
667		try:
668	<	pippo = sbi.copy( source_file , dest_file )
669	<	if self.protocol == 'srm' : self.checkSize( sbi, filetocopy )
670	<	except Exception, ex:
671	<	msg = ''
672	<	if self.debug : msg = str(ex)+'\n'
673	<	msg = "Problem copying %s file with %s command"%( filetocopy, protocol )
668	>	sbi.copy( source_file , dest_file , opt = copy_option, tout = self.subprocesstimeout['copy'])
669	>	except TransferException, ex:
670	>	msg  = "Problem copying %s file" % filetocopy
671	>	msg += str(ex)
672	>	if self.debug :
673	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
674	>	dbgmsg += '\t'+str(ex.output)+'\n'
675	>	print dbgmsg
676		ErCode = '60307'
677	<	#print msg
678	<
677	>	except WrongOption, ex:
678	>	msg  = "Problem copying %s file" % filetocopy
679	>	msg += str(ex)
680	>	if self.debug :
681	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
682	>	dbgmsg += '\t'+str(ex.output)+'\n'
683	>	print dbgmsg
684	>	except SizeZeroException, ex:
685	>	msg  = "Problem copying %s file" % filetocopy
686	>	msg += str(ex)
687	>	if self.debug :
688	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
689	>	dbgmsg += '\t'+str(ex.output)+'\n'
690	>	print dbgmsg
691	>	ErCode = '60307'
692	>	## when the client commands are not found (wrong env or really missing)
693	>	except MissingCommand, ex:
694	>	ErCode = '10041'
695	>	msg  = "Problem copying %s file" % filetocopy
696	>	msg += str(ex)
697	>	if self.debug :
698	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
699	>	dbgmsg += '\t'+str(ex.output)+'\n'
700	>	print dbgmsg
701	>	except AuthorizationException, ex:
702	>	ErCode = '60307'
703	>	msg  = "Problem copying %s file" % filetocopy
704	>	msg += str(ex)
705	>	if self.debug :
706	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
707	>	dbgmsg += '\t'+str(ex.output)+'\n'
708	>	print dbgmsg
709	>	except SEAPITimeout, ex:
710	>	ErCode = '60317'
711	>	msg  = "Problem copying %s file" % filetocopy
712	>	msg += str(ex)
713	>	if self.debug :
714	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
715	>	dbgmsg += '\t'+str(ex.output)+'\n'
716	>	print dbgmsg
717	>
718	>	if ErCode == '0' and protocol.find('srmv') == 0:
719	>	remote_file_size = -1
720	>	local_file_size = os.path.getsize( source_file )
721	>	try:
722	>	remote_file_size = sbi_dest.getSize( dest_file, opt=option, tout = self.subprocesstimeout['size'] )
723	>	if self.debug : print '\t Check of remote size succeded for file %s\n'%dest_file
724	>	except TransferException, ex:
725	>	msg  = "Problem checking the size of %s file" % filetocopy
726	>	msg += str(ex)
727	>	if self.debug :
728	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
729	>	dbgmsg += '\t'+str(ex.output)+'\n'
730	>	print dbgmsg
731	>	ErCode = '60307'
732	>	except WrongOption, ex:
733	>	msg  = "Problem checking the size of %s file" % filetocopy
734	>	msg += str(ex)
735	>	if self.debug :
736	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
737	>	dbgmsg += '\t'+str(ex.output)+'\n'
738	>	print dbgmsg
739	>	ErCode = '60307'
740	>	if local_file_size != remote_file_size:
741	>	msg = "File size dosn't match: local size = %s ; remote size = %s " % (local_file_size, remote_file_size)
742	>	ErCode = '60307'
743	>
744	>	if ErCode != '0':
745	>	try :
746	>	self.removeFile( sbi_dest, dest_file, option )
747	>	except Exception, ex:
748	>	msg += '\n'+str(ex)
749		return ErCode, msg
750	<
751	<	'''
752	<	def checkSize()
750	>
751	>	def removeFile( self, sbi_dest, filetocopy, option ):
752	>	"""
753	>	"""
754	>	if self.debug : print 'removeFile():\n'
755	>	f_tocopy=filetocopy
756	>	if self.dest_prot != 'local':f_tocopy = os.path.basename(filetocopy)
757	>	try:
758	>	sbi_dest.delete( f_tocopy, opt=option, tout = self.subprocesstimeout['delete'] )
759	>	if self.debug : '\t deletion of file %s succeeded\n'%str(filetocopy)
760	>	except OperationException, ex:
761	>	msg  ='ERROR: problems removing partially staged file %s'%filetocopy
762	>	msg += str(ex)
763	>	if self.debug :
764	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
765	>	dbgmsg += '\t'+str(ex.output)+'\n'
766	>	print dbgmsg
767	>	raise Exception(msg)
768	>
769	>	return
770	>
771	>	def updateReport(self, file, erCode, reason):
772		"""
773	<	Using srm needed a check of the ouptut file size.
773	>	Update the final stage out infos
774		"""
775	<
776	<	echo "--> remoteSize = $remoteSize"
777	<	## for local file
778	<	localSize=$(stat -c%s "$path_out_file")
779	<	echo "-->  localSize = $localSize"
780	<	if [ $localSize != $remoteSize ]; then
781	<	echo "Local fileSize $localSize does not match remote fileSize $remoteSize"
782	<	echo "Copy failed: removing remote file $destination"
783	<	srmrm $destination
784	<	cmscp_exit_status=60307
785	<
786	<
787	<	echo "Problem copying $path_out_file to $destination with srmcp command"
788	<	StageOutExitStatusReason='remote and local file dimension not match'
789	<	echo "StageOutReport = `cat ./srmcp.report`"
790	<	'''
791	<	def backup(self):
792	<	"""
793	<	Check infos from TFC using existing api obtaining:
794	<	1)destination
795	<	2)protocol
775	>	jobStageInfo={}
776	>	jobStageInfo['erCode']=erCode
777	>	jobStageInfo['reason']=reason
778	>	if not self.params['for_lfn']: self.params['for_lfn']=''
779	>	if not self.params['se_name']: self.params['se_name']=''
780	>	if not self.hostname: self.hostname=''
781	>	if (erCode != '0') and (erCode != '60308'):
782	>	jobStageInfo['for_lfn']='/copy_problem/'
783	>	else:
784	>	jobStageInfo['for_lfn']=self.params['for_lfn']
785	>	jobStageInfo['se_name']=self.params['se_name']
786	>	jobStageInfo['endpoint']=self.hostname
787	>	### ADDING SURLFORGRID FOR COPYDATA
788	>	if not self.params['surl_for_grid']: self.params['surl_for_grid']=''
789	>	jobStageInfo['surl_for_grid']=self.params['surl_for_grid']
790	>	#####
791	>	report = { file : jobStageInfo}
792	>	return report
793	>
794	>	def finalReport( self , results ):
795	>	"""
796	>	It a list of LFNs for each SE where data are stored.
797	>	allow "crab -copyLocal" or better "crab -copyOutput". TO_DO.
798		"""
799	+
800	+	outFile = 'cmscpReport.sh'
801	+	if os.getenv("RUNTIME_AREA"):
802	+	#print "RUNTIME_AREA = ", os.getenv("RUNTIME_AREA")
803	+	outFile = "%s/cmscpReport.sh"%os.getenv("RUNTIME_AREA")
804	+	#print "--->>> outFile = ", outFile
805	+	fp = open(outFile, "w")
806	+
807	+	cmscp_exit_status = 0
808	+	txt = ''
809	+	for file, dict in results.iteritems():
810	+	reason = str(dict['reason'])
811	+	if str(reason).find("'") > -1:
812	+	reason = " ".join(reason.split("'"))
813	+	reason="'%s'"%reason
814	+	if file:
815	+	if dict['for_lfn']=='':
816	+	lfn = '${LFNBaseName}'+os.path.basename(file)
817	+	se  = '$SE'
818	+	LFNBaseName = '$LFNBaseName'
819	+	else:
820	+	lfn = dict['for_lfn']+os.path.basename(file)
821	+	se = dict['se_name']
822	+	LFNBaseName = os.path.dirname(lfn)
823	+	if (LFNBaseName[-1] != '/'):
824	+	LFNBaseName = LFNBaseName + '/'
825	+
826	+
827	+	txt += 'echo "Report for File: '+file+'"\n'
828	+	txt += 'echo "LFN: '+lfn+'"\n'
829	+	txt += 'echo "StorageElement: '+se+'"\n'
830	+	txt += 'echo "StageOutExitStatusReason = %s" | tee -a $RUNTIME_AREA/$repo\n'%reason
831	+	txt += 'echo "StageOutSE = '+se+'" >> $RUNTIME_AREA/$repo\n'
832	+
833	+
834	+	if dict['erCode'] != '0':
835	+	cmscp_exit_status = dict['erCode']
836	+	else:
837	+	txt += 'echo "StageOutExitStatusReason = %s" | tee -a $RUNTIME_AREA/$repo\n'%reason
838	+	cmscp_exit_status = dict['erCode']
839	+	txt += '\n'
840	+	txt += 'export StageOutExitStatus='+str(cmscp_exit_status)+'\n'
841	+	txt += 'echo "StageOutExitStatus = '+str(cmscp_exit_status)+'" | tee -a $RUNTIME_AREA/$repo\n'
842	+	fp.write(str(txt))
843	+	fp.close()
844	+	if self.debug:
845	+	print '--- reading cmscpReport.sh: \n'
846	+	lp = open(outFile, "r")
847	+	content = lp.read()
848	+	lp.close()
849	+	print "    content = ", content
850	+	print '--- end reading cmscpReport.sh'
851		return
852
386	–	def usage(self):
853
854	<	msg="""
855	<	required parameters:
856	<	--source :: REMOTE           :
857	<	--dest   :: REMOTE           :
858	<	--debug             :
859	<	--inFile :: absPath : or name NOT RELATIVE PATH
860	<	--outFIle :: onlyNAME : NOT YET SUPPORTED
861	<
862	<	optional parameters
863	<	"""
864	<	return msg
854	>	def usage():
855	>
856	>	msg="""
857	>	cmscp:
858	>	safe copy of local file  to/from remote SE via lcg_cp/srmcp,
859	>	including success checking  version also for CAF using rfcp command to copy the output to SE
860	>
861	>	accepted parameters:
862	>	source           =
863	>	destination      =
864	>	inputFileList    =
865	>	outputFileList   =
866	>	protocol         =
867	>	option           =
868	>	middleware       =
869	>	srm_version      =
870	>	destinationDir   =
871	>	lfn=             =
872	>	local_stage      =  activate stage fall back
873	>	debug            =  activate verbose print out
874	>	help             =  print on line man and exit
875	>
876	>	mandatory:
877	>	* "source" and/or "destination" must always be defined
878	>	* either "middleware" or "protocol" must always be defined
879	>	* "inputFileList" must always be defined
880	>	* if "local_stage" = 1 also  "lfn" must be defined
881	>	"""
882	>	print msg
883	>
884	>	return
885	>
886	>	def HelpOptions(opts=[]):
887	>	"""
888	>	Check otps, print help if needed
889	>	prepare dict = { opt : value }
890	>	"""
891	>	dict_args = {}
892	>	if len(opts):
893	>	for opt, arg in opts:
894	>	dict_args[opt.split('--')[1]] = arg
895	>	if opt in ('-h','-help','--help') :
896	>	usage()
897	>	sys.exit(0)
898	>	return dict_args
899	>	else:
900	>	usage()
901	>	sys.exit(0)
902
903		if __name__ == '__main__' :
904	+
905	+	import getopt
906	+
907	+	allowedOpt = ["source=", "destination=", "inputFileList=", "outputFileList=", \
908	+	"protocol=","option=", "middleware=", "srm_version=", \
909	+	"destinationDir=", "for_lfn=", "local_stage", "debug", "help", "se_name="]
910	+	try:
911	+	opts, args = getopt.getopt( sys.argv[1:], "", allowedOpt )
912	+	except getopt.GetoptError, err:
913	+	print err
914	+	HelpOptions()
915	+	sys.exit(2)
916	+
917	+	dictArgs = HelpOptions(opts)
918		try:
919	<	cmscp_ = cmscp(sys.argv[1:])
919	>	cmscp_ = cmscp(dictArgs)
920		cmscp_.run()
921	<	except:
922	<	pass
921	>	except Exception, ex :
922	>	print str(ex)
923

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines