ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/cmscp.py

Comparing COMP/CRAB/python/cmscp.py (file contents):
Revision 1.4 by spiga, Fri Sep 26 11:15:35 2008 UTC vs.
Revision 1.94 by belforte, Wed Jan 9 14:59:47 2013 UTC

#	Line 1 | Line 1
1		#!/usr/bin/env python
2	<
3	<	import sys, getopt, string
4	<	import os, popen2
5	<	from ProdCommon.Storage.SEAPI.SElement import SElement, FullPath
2	>	import sys, os
3	>	import time, random
4	>	try:
5	>	import json
6	>	except:
7	>	import simplejson as json
8	>	from ProdCommon.Storage.SEAPI.SElement import SElement, FullPath
9		from ProdCommon.Storage.SEAPI.SBinterface import *
10	+	from ProdCommon.Storage.SEAPI.Exceptions import *
11	+	from ProdCommon.FwkJobRep.SiteLocalConfig import loadSiteLocalConfig
12
13
14
15		class cmscp:
16	<	def __init__(self, argv):
16	>	def __init__(self, args):
17		"""
18		cmscp
19	<
15	<	safe copy of local file in current directory to remote SE via lcg_cp/srmcp,
19	>	safe copy of local file  to/from remote SE via lcg_cp/srmcp,
20		including success checking  version also for CAF using rfcp command to copy the output to SE
21		input:
22		$1 middleware (CAF, LSF, LCG, OSG)
23		$2 local file (the absolute path of output file or just the name if it's in top dir)
24		$3 if needed: file name (the output file name)
25		$5 remote SE (complete endpoint)
26	<	$6 srm version
26	>	$6 srm version
27	>	--for_lfn $LFNBaseName
28		output:
29		return 0 if all ok
30		return 60307 if srmcp failed
31		return 60303 if file already exists in the SE
32		"""
33	<	#set default
33	>	self.params = {"source":'', "destination":'','destinationDir':'', "inputFileList":'', "outputFileList":'', \
34	>	"protocol":'', "option":'', "middleware":'', "srm_version":'srmv2', "for_lfn":'', "se_name":'', "surl_for_grid":''}
35		self.debug = 0
36	<	self.source = ''
37	<	self.destination = ''
38	<	self.file_to_copy = []
39	<	self.remote_file_name = []
40	<	self.protocol = ''
41	<	self.middleware = ''
42	<	self.srmv = ''
43	<	self.lcgOpt='-b -D srmv2 --vo cms -t 2400 --verbose'
44	<	self.opt=''
45	<	try:
46	<	opts, args = getopt.getopt(argv, "", ["source=", "destination=", "inputFileList=", "outputFileList=", \
47	<	"protocol=", "middleware=", "srm_version=", "debug", "help"])
48	<	except getopt.GetoptError:
43	<	print self.usage()
44	<	sys.exit(2)
45	<
46	<	self.setAndCheck(opts)
47	<
36	>	#### for fallback copy
37	>	self.local_stage = 0
38	>	self.params.update( args )
39	>	## timeout needed for subprocess command of SEAPI
40	>	## they should be a bit higher then the corresponding passed by command line
41	>	## default values
42	>	self.subprocesstimeout = { \
43	>	'copy':   3600, \
44	>	'exists': 1200, \
45	>	'delete': 1200, \
46	>	'size':   1200 \
47	>	}
48	>
49		return
50
51	<	def setAndCheck( self, opts ):
51	>	def processOptions( self ):
52		"""
53	<	Set and check command line parameter
53	>	check command line parameter
54		"""
55	<	if not opts :
56	<	print self.usage()
57	<	sys.exit()
58	<	for opt, arg in opts :
59	<	if opt  == "--help" :
60	<	print self.usage()
61	<	sys.exit()
62	<	elif opt == "--debug" :
63	<	self.debug = 1
64	<	elif opt == "--source" :
65	<	self.source = arg
66	<	elif opt == "--destination":
67	<	self.destination = arg
68	<	elif opt == "--inputFileList":
68	<	infile = arg
69	<	elif opt == "--outputFileList":
70	<	out_file
71	<	elif opt == "--protocol":
72	<	self.protocol = arg
73	<	elif opt == "--middleware":
74	<	self.middleware = arg
75	<	elif opt == "--srm_version":
76	<	self.srmv = arg
77	<
78	<	# source and dest cannot be undefined at same time
79	<	if self.source == '' and self.destination == '':
80	<	print self.usage()
81	<	sys.exit()
82	<	# if middleware is not defined --> protocol cannot be empty
83	<	if self.middleware == '' and self.protocol == '':
84	<	print self.usage()
85	<	sys.exit()
86	<	# input file must be defined
87	<	if infile == '':
88	<	print self.usage()
89	<	sys.exit()
55	>	if 'help' in self.params.keys(): HelpOptions()
56	>	if 'debug' in self.params.keys(): self.debug = 1
57	>	if 'local_stage' in self.params.keys(): self.local_stage = 1
58	>
59	>	# source and dest cannot be undefined at same time
60	>	if not self.params['source']  and not self.params['destination'] :
61	>	HelpOptions()
62	>	# if middleware is not defined --> protocol cannot be empty
63	>	if not self.params['middleware'] and not self.params['protocol'] :
64	>	HelpOptions()
65	>
66	>	# input file must be defined
67	>	if not self.params['inputFileList'] :
68	>	HelpOptions()
69		else:
70	<	if infile.find(','):
71	<	[self.file_to_copy.append(x.strip()) for x in infile.split(',')]
72	<	else:
73	<	self.file_to_copy.append(infile)
74	<
75	<	## TO DO:
97	<	#### add check for outFiles
98	<	#### add map {'inFileNAME':'outFileNAME'} to change out name
70	>	file_to_copy=[]
71	>	if self.params['inputFileList'].find(','):
72	>	[file_to_copy.append(x.strip()) for x in self.params['inputFileList'].split(',')]
73	>	else:
74	>	file_to_copy.append(self.params['inputFileList'])
75	>	self.params['inputFileList'] = file_to_copy
76
77	<	return
77	>	if not self.params['for_lfn'] and self.local_stage == 1 : HelpOptions()
78	>
79
80	<	def run( self ):
103	<	"""
104	<	Check if running on UI (no $middleware) or
105	<	on WN (on the Grid), and take different action
80	>	def run( self ):
81		"""
82	<	if self.middleware :
83	<	results = self.stager()
82	>	Check if running on UI (no $middleware) or
83	>	on WN (on the Grid), and take different action
84	>	"""
85	>	self.processOptions()
86	>	if self.debug: print 'calling run() : \n'
87	>	# stage out from WN
88	>	if self.params['middleware'] :
89	>	results = self.stager(self.params['middleware'],self.params['inputFileList'])
90	>	self.writeJsonFile(results)
91	>	self.finalReport(results)
92	>	# Local interaction with SE
93		else:
94	<	results = self.copy( self.file_to_copy, self.protocol , self.opt)
94	>	results = self.copy(self.params['inputFileList'], self.params['protocol'], self.params['option'] )
95	>	self.writeJsonFile(results)
96	>	return results
97	>
98	>	def writeJsonFile( self, results ):
99	>	"""
100	>	write a json file containing copy results for each file
101	>	"""
102	>	if self.debug:
103	>	print 'in writeJsonFile() : \n'
104	>	print "---->>>> in writeJsonFile results =  ", results
105	>	jsonOut = "resultCopyFile"
106	>	if os.getenv("RUNTIME_AREA"):
107	>	jsonOut = "%s/resultCopyFile"%os.getenv("RUNTIME_AREA")
108	>	fp = open(jsonOut, 'w')
109	>	json.dump(results, fp)
110	>	fp.close()
111	>	if self.debug:
112	>	print '    reading resultCopyFile : \n'
113	>	lp = open(jsonOut, "r")
114	>	inputDict = json.load(lp)
115	>	lp.close()
116	>	print "    inputDict = ", inputDict
117	>	return
118
119	<	self.finalReport(results,self.middleware)
119	>	def checkLcgUtils( self ):
120	>	"""
121	>	_checkLcgUtils_
122	>	check the lcg-utils version and report
123	>	"""
124	>	import commands
125	>	cmd = "lcg-cp --version | grep lcg_util"
126	>	status, output = commands.getstatusoutput( cmd )
127	>	num_ver = -1
128	>	if output.find("not found") == -1 or status == 0:
129	>	temp = output.split("-")
130	>	version = ""
131	>	if len(temp) >= 2:
132	>	version = output.split("-")[1]
133	>	temp = version.split(".")
134	>	if len(temp) >= 1:
135	>	num_ver = int(temp[0])*10
136	>	num_ver += int(temp[1])
137	>	return num_ver
138
139	<	return
115	<
116	<	def setProtocol( self ):
139	>	def setProtocol( self, middleware ):
140		"""
141		define the allowed potocols based on $middlware
142	<	which depend on scheduler
142	>	which depend on scheduler
143		"""
144	<	if self.middleware.lower() in ['osg','lcg']:
145	<	supported_protocol = {'srm-lcg': self.lcgOpt }#,
146	<	#   'srmv2' : '' }
147	<	elif self.middleware.lower() in ['lsf','caf']:
148	<	supported_protocol = {'rfio': '' }
144	>	# default To be used with "middleware"
145	>	if self.debug:
146	>	print 'setProtocol() :\n'
147	>	print '\tmiddleware =  %s utils \n'%middleware
148	>
149	>	lcgOpt={'srmv1':'-b -D srmv1  -t 2400 --verbose',
150	>	'srmv2':'-b -D srmv2  -t 2400 --verbose'}
151	>	if self.checkLcgUtils() >= 17:
152	>	lcgOpt={'srmv1':'-b -D srmv1 --srm-timeout 2400 --sendreceive-timeout 2400 --connect-timeout 300 --verbose',
153	>	'srmv2':'-b -D srmv2 --srm-timeout 2400 --sendreceive-timeout 2400 --connect-timeout 300 --verbose'}
154	>
155	>	srmOpt={'srmv1':' -report ./srmcp.report -retry_timeout 480000 -retry_num 3 -streams_num=1 ',
156	>	'srmv2':' -report=./srmcp.report -retry_timeout=480000 -retry_num=3 -storagetype=permanent '}
157	>	rfioOpt=''
158	>	#### FEDE FOR XROOTD #########
159	>	xrootdOpt=''
160	>	#############################
161	>
162	>	supported_protocol = None
163	>	if middleware.lower() in ['osg','lcg','condor','sge']:
164	>	supported_protocol = [('srm-lcg',lcgOpt[self.params['srm_version']])]#,\
165	>	#  (self.params['srm_version'],srmOpt[self.params['srm_version']])]
166	>	#elif middleware.lower() in ['lsf','caf']:
167	>	elif middleware.lower() in ['lsf']:
168	>	supported_protocol = [('rfio',rfioOpt)]
169	>	elif middleware.lower() in ['pbs']:
170	>	supported_protocol = [('rfio',rfioOpt),('local','')]
171	>	elif middleware.lower() in ['arc']:
172	>	supported_protocol = [('srmv2','-debug'),('srmv1','-debug')]
173	>	#### FEDE FOR XROOTD ##########
174	>	elif middleware.lower() in ['caf']:
175	>	if self.params['protocol']:
176	>	supported_protocol = [(self.params['protocol'], '')]
177	>	else:
178	>	supported_protocol = [('rfio',rfioOpt)]
179	>	#######################################
180		else:
181	<	## here we can add support for any kind of protocol,
181	>	## here we can add support for any kind of protocol,
182		## maybe some local schedulers need something dedicated
183		pass
184	+
185	+	# force lstore protocol for Vanderbilt, waiting for better
186	+	# long term solution
187	+	if os.path.exists('/usr/local/cms-stageout') and \
188	+	( os.uname()[1].endswith('.vampire') or \
189	+	os.uname()[1].endwith('vanderbilt.edu') ):
190	+	print "*** I am at Vanderbilt. Trying lstore first ***"
191	+	supported_protocol.insert(0, ('lstore','') )
192	+
193		return supported_protocol
194
195	<	def stager( self ):
195	>
196	>	def checkCopy (self, copy_results, len_list_files, prot):
197	>	"""
198	>	Checks the status of copy and update result dictionary
199	>	"""
200	>
201	>	list_retry = []
202	>	list_not_existing = []
203	>	list_already_existing = []
204	>	list_fallback = []
205	>	list_ok = []
206	>
207	>	if self.debug:
208	>	print 'in checkCopy() :\n'
209	>
210	>	for file, dict in copy_results.iteritems():
211	>	er_code = dict['erCode']
212	>	if er_code == '0':
213	>	list_ok.append(file)
214	>	reason = 'Copy succedeed with %s utils'%prot
215	>	dict['reason'] = reason
216	>	elif er_code == '60308':
217	>	list_fallback.append( file )
218	>	reason = 'Copy succedeed with %s utils'%prot
219	>	dict['reason'] = reason
220	>	elif er_code == '60302':
221	>	list_not_existing.append( file )
222	>	elif er_code == '60303':
223	>	list_already_existing.append( file )
224	>	else :
225	>	list_retry.append( file )
226	>
227	>	if self.debug:
228	>	print "\t file %s \n"%file
229	>	print "\t dict['erCode'] %s \n"%dict['erCode']
230	>	print "\t dict['reason'] %s \n"%dict['reason']
231	>
232	>	upDict = self.updateReport(file, er_code, dict['reason'])
233	>
234	>	copy_results.update(upDict)
235	>
236	>	msg = ''
237	>	if len(list_ok) != 0:
238	>	msg += '\tCopy of %s succedeed with %s utils\n'%(str(list_ok),prot)
239	>	if len(list_ok) != len_list_files :
240	>	if len(list_fallback)!=0:
241	>	msg += '\tCopy of %s succedeed with %s utils in the fallback SE\n'%(str(list_fallback),prot)
242	>	if len(list_retry)!=0:
243	>	msg += '\tCopy of %s failed using %s for files \n'%(str(list_retry),prot)
244	>	if len(list_not_existing)!=0:
245	>	msg += '\tCopy of %s failed using %s : files not found \n'%(str(list_not_existing),prot)
246	>	if len(list_already_existing)!=0:
247	>	msg += '\tCopy of %s failed using %s : files already existing\n'%(str(list_already_existing),prot)
248	>	if self.debug : print msg
249	>	return copy_results, list_ok, list_retry, list_fallback
250	>
251	>	def check_for_retry_localSE (self, copy_results):
252	>	"""
253	>	Checks the status of copy and create the list of file to copy to CloseSE
254	>	"""
255	>	list_retry_localSE = []
256	>
257	>	if self.debug:
258	>	print 'in check_for_retry_localSE() :\n'
259	>	print "\t results in check local = ", copy_results
260	>	for file, dict in copy_results.iteritems():
261	>	er_code = dict['erCode']
262	>	if er_code != '0' and  er_code != '60302' and er_code != '60308':
263	>	list_retry_localSE.append( file )
264	>
265	>	if self.debug:
266	>	print "\t file %s \n"%file
267	>	print "\t dict['erCode'] %s \n"%dict['erCode']
268	>	print "\t dict['reason'] %s \n"%dict['reason']
269	>
270	>	return list_retry_localSE
271	>
272	>
273	>	def LocalCopy(self, list_retry, results):
274	>	"""
275	>	Tries the stage out to the CloseSE
276	>	"""
277	>	if self.debug:
278	>	print 'in LocalCopy() :\n'
279	>	print '\t list_retry %s utils \n'%list_retry
280	>	print '\t len(list_retry) %s \n'%len(list_retry)
281	>
282	>	list_files = list_retry
283	>	self.params['inputFileList']=list_files
284	>
285	>	### copy backup
286	>	from ProdCommon.FwkJobRep.SiteLocalConfig import loadSiteLocalConfig
287	>	siteCfg = loadSiteLocalConfig()
288	>	catalog = siteCfg.localStageOut.get("catalog", None)
289	>	tfc = siteCfg.trivialFileCatalog()
290	>	seName = siteCfg.localStageOutSEName()
291	>	if seName is None:
292	>	seName = ""
293	>	option = siteCfg.localStageOutOption()
294	>	if option is None:
295	>	option = ""
296	>	implName = siteCfg.localStageOutCommand()
297	>	if implName is None:
298	>	implName = ""
299	>
300	>	if (implName == 'srm'):
301	>	protocol = 'srmv2'
302	>	elif (implName == 'srmv2-lcg'):
303	>	protocol = 'srm-lcg'
304	>	option = option + ' -b -D srmv2 '
305	>	elif (implName == 'rfcp-CERN'):
306	>	protocol = 'rfio'
307	>	elif (implName == 'rfcp'):
308	>	protocol = 'rfio'
309	>	elif (implName == 'cp'):
310	>	protocol = 'local'
311	>	else: protocol = implName
312	>
313	>	self.params['protocol']=protocol
314	>	self.params['option']=option
315	>
316	>	if self.debug:
317	>	print '\t siteCFG %s \n'%siteCfg
318	>	print '\t catalog %s \n'%catalog
319	>	print '\t tfc %s '%tfc
320	>	print '\t fallback seName %s \n'%seName
321	>	print "\t fallback protocol %s \n"%protocol
322	>	print "\t fallback option %s \n"%option
323	>	print "\t self.params['inputFileList'] %s \n"%self.params['inputFileList']
324	>
325	>	if (str(self.params['for_lfn']).find("/store/") == 0):
326	>	temp = str(self.params['for_lfn']).replace("/store/","/store/temp/",1)
327	>	self.params['for_lfn']= temp
328	>
329	>	if ( self.params['for_lfn'][-1] != '/' ) : self.params['for_lfn'] = self.params['for_lfn'] + '/'
330	>
331	>	file_backup=[]
332	>	file_backup_surlgrid=[]
333	>	for input in self.params['inputFileList']:
334	>	file = self.params['for_lfn'] + os.path.basename(input)
335	>	surl = tfc.matchLFN(tfc.preferredProtocol, file)
336	>
337	>	###### FEDE TEST_FOR_SURL_GRID
338	>	surl_for_grid = tfc.matchLFN('srmv2', file)
339	>	if (surl_for_grid == None):
340	>	surl_for_grid = tfc.matchLFN('srmv2-lcg', file)
341	>	if (surl_for_grid == None):
342	>	surl_for_grid = tfc.matchLFN('srm', file)
343	>	if surl_for_grid:
344	>	file_backup_surlgrid.append(surl_for_grid)
345	>	######
346	>
347	>	file_backup.append(surl)
348	>	if self.debug:
349	>	print '\t for_lfn %s \n'%self.params['for_lfn']
350	>	print '\t file %s \n'%file
351	>	print '\t surl %s \n'%surl
352	>
353	>	destination=os.path.dirname(file_backup[0])
354	>	if ( destination[-1] != '/' ) : destination = destination + '/'
355	>	self.params['destination']=destination
356	>
357	>	self.params['se_name']=seName
358	>
359	>	######
360	>	if (len(file_backup_surlgrid)>0) :
361	>	surl_for_grid=os.path.dirname(file_backup_surlgrid[0])
362	>	if ( surl_for_grid[-1] != '/' ) : surl_for_grid = surl_for_grid + '/'
363	>	else:
364	>	surl_for_grid=''
365	>
366	>	print "surl_for_grid = ", surl_for_grid
367	>	self.params['surl_for_grid']=surl_for_grid
368	>	#####
369	>
370	>	if self.debug:
371	>	print '\tIn LocalCopy trying the stage out with: \n'
372	>	print "\tself.params['destination'] %s \n"%self.params['destination']
373	>	print "\tself.params['protocol'] %s \n"%self.params['protocol']
374	>	print "\tself.params['option'] %s \n"%self.params['option']
375	>
376	>	localCopy_results = self.copy( self.params['inputFileList'], self.params['protocol'], self.params['option'], backup='yes' )
377	>
378	>	if localCopy_results.keys() == [''] or localCopy_results.keys() == '' :
379	>	results.update(localCopy_results)
380	>	else:
381	>	localCopy_results, list_ok, list_retry, list_fallback = self.checkCopy(localCopy_results, len(list_files), self.params['protocol'])
382	>
383	>	results.update(localCopy_results)
384	>	if self.debug:
385	>	print "\t localCopy_results = %s \n"%localCopy_results
386	>	return results
387	>
388	>	def stager( self, middleware, list_files ):
389		"""
390		Implement the logic for remote stage out
391		"""
136	–	protocols = self.setProtocol()
137	–	count=0
138	–	list_files = self.file_to_copy
139	–	results={}
140	–	for prot in protocols.keys():
141	–	if self.debug: print 'Trying stage out with %s utils \n'%prot
142	–	copy_results = self.copy( list_files, prot, protocols[prot] )
143	–	list_retry = []
144	–	list_existing = []
145	–	list_ok = []
146	–	for file, dict in copy_results.iteritems():
147	–	er_code = dict['erCode']
148	–	if er_code == '60307': list_retry.append( file )
149	–	elif er_code == '60303': list_existing.append( file )
150	–	else:
151	–	list_ok.append(file)
152	–	reason = 'Copy succedeed with %s utils'%prot
153	–	upDict = self.updateReport(file, er_code, reason)
154	–	copy_results.update(upDict)
155	–	results.update(copy_results)
156	–	if len(list_ok) != 0:
157	–	msg = 'Copy of %s succedeed with %s utils\n'%(str(list_ok),prot)
158	–	# print msg
159	–	if len(list_ok) == len(list_files) :
160	–	break
161	–	else:
162	–	#       print 'Copy of files %s failed using %s...\n'%(str(list_retry)+str(list_existing),prot)
163	–	if len(list_retry): list_files = list_retry
164	–	else: break
165	–	count =+1
392
393	<	#### TODO Daniele
394	<	#check is something fails and created related dict
395	<	#      backup = self.analyzeResults(results)
396	<
397	<	#      if backup :
398	<	#          msg = 'WARNING: backup logic is under implementation\n'
399	<	#          #backupDict = self.backup()
400	<	#          ### NOTE: IT MUST RETURN a DICT contains also LFN and SE Name
401	<	#          results.update(backupDict)
402	<	#          print msg
393	>	if self.debug:
394	>	print 'stager() :\n'
395	>	print '\tmiddleware %s\n'%middleware
396	>	print '\tlist_files %s\n'%list_files
397	>
398	>	results={}
399	>	for prot, opt in self.setProtocol( middleware ):
400	>	if self.debug:
401	>	print '\tTrying the stage out with %s utils \n'%prot
402	>	print '\tand options %s\n'%opt
403	>
404	>	copy_results = self.copy( list_files, prot, opt )
405	>	if copy_results.keys() == [''] or copy_results.keys() == '' :
406	>	results.update(copy_results)
407	>	else:
408	>	copy_results, list_ok, list_retry, list_fallback = self.checkCopy(copy_results, len(list_files), prot)
409	>	results.update(copy_results)
410	>	if len(list_ok) == len(list_files) :
411	>	break
412	>	if len(list_retry):
413	>	list_files = list_retry
414	>	#### FEDE added ramdom time before the retry copy with other protocol
415	>	sec =  240 * random.random()
416	>	sec = sec + 60
417	>	time.sleep(sec)
418	>	else: break
419	>
420	>	if self.local_stage:
421	>	list_retry_localSE = self.check_for_retry_localSE(results)
422	>	if len(list_retry_localSE):
423	>	if self.debug:
424	>	print "\t list_retry_localSE %s \n"%list_retry_localSE
425	>	results = self.LocalCopy(list_retry_localSE, results)
426	>
427	>	if self.debug:
428	>	print "\t results %s \n"%results
429		return results
430
431		def initializeApi(self, protocol ):
432		"""
433	<	Instantiate storage interface
433	>	Instantiate storage interface
434		"""
435	<	source_prot = protocol
436	<	dest_prot = protocol
437	<	if self.source == '' : source_prot = 'local'
438	<	Source_SE  = self.storageInterface( self.source, source_prot )
439	<	if self.destination == '' : dest_prot = 'local'
440	<	Destination_SE = self.storageInterface( self.destination, dest_prot )
435	>	if self.debug : print 'initializeApi() :\n'
436	>	self.source_prot = protocol
437	>	self.dest_prot = protocol
438	>	if not self.params['source'] : self.source_prot = 'local'
439	>	Source_SE  = self.storageInterface( self.params['source'], self.source_prot )
440	>	if not self.params['destination'] :
441	>	self.dest_prot = 'local'
442	>	Destination_SE = self.storageInterface( self.params['destinationDir'], self.dest_prot )
443	>	else:
444	>	Destination_SE = self.storageInterface( self.params['destination'], self.dest_prot )
445
446		if self.debug :
447	<	print '(source=%s,  protocol=%s)'%(self.source, source_prot)
448	<	print '(destination=%s,  protocol=%s)'%(self.destination, dest_prot)
447	>	msg  = '\t(source=%s,  protocol=%s)'%(self.params['source'], self.source_prot)
448	>	msg += '\t(destination=%s,  protocol=%s)'%(self.params['destination'], self.dest_prot)
449	>	msg += '\t(destinationDir=%s,  protocol=%s)'%(self.params['destinationDir'], self.dest_prot)
450	>	print msg
451
452		return Source_SE, Destination_SE
453
454	<	def copy( self, list_file, protocol, opt):
454	>	def copy( self, list_file, protocol, options, backup='no' ):
455		"""
456	<	Make the real file copy using SE API
456	>	Make the real file copy using SE API
457		"""
458	+	msg = ""
459	+	results = {}
460		if self.debug :
461	<	print 'copy(): using %s protocol'%protocol
462	<	Source_SE, Destination_SE = self.initializeApi( protocol )
463	<
464	<	# create remote dir
465	<	if protocol in ['gridftp','rfio']:
466	<	self.createDir( Destination_SE, protocol )
467	<
461	>	msg  = 'copy() :\n'
462	>	msg += '\tusing %s protocol\n'%protocol
463	>	msg += '\tusing %s options\n'%options
464	>	msg += '\tlist_file %s\n'%list_file
465	>	print msg
466	>	try:
467	>	Source_SE, Destination_SE = self.initializeApi( protocol )
468	>	except Exception, ex:
469	>	for filetocopy in list_file:
470	>	results.update( self.updateReport(filetocopy, '-1', str(ex)))
471	>	return results
472	>
473	>	self.hostname = Destination_SE.hostname
474	>	if Destination_SE.protocol in ['gridftp','rfio','srmv2','hadoop','lstore','local']:
475	>	try:
476	>	self.createDir( Destination_SE, Destination_SE.protocol )
477	>	except OperationException, ex:
478	>	for filetocopy in list_file:
479	>	results.update( self.updateReport(filetocopy, '60316', str(ex)))
480	>	return results
481	>	## when the client commands are not found (wrong env or really missing)
482	>	except MissingCommand, ex:
483	>	msg = "ERROR %s %s" %(str(ex), str(ex.detail))
484	>	for filetocopy in list_file:
485	>	results.update( self.updateReport(filetocopy, '10041', msg))
486	>	return results
487	>	except Exception, ex:
488	>	msg = "ERROR %s" %(str(ex))
489	>	for filetocopy in list_file:
490	>	results.update( self.updateReport(filetocopy, '-1', msg))
491	>	return results
492		## prepare for real copy  ##
493	<	sbi = SBinterface( Source_SE, Destination_SE )
494	<	sbi_dest = SBinterface(Destination_SE)
495	<
496	<	results = {}
497	<	## loop over the complete list of files
498	<	for filetocopy in list_file:
499	<	if self.debug : print 'start real copy for %s'%filetocopy
500	<	ErCode, msg = self.checkFileExist( sbi_dest, os.path.basename(filetocopy) )
501	<	if ErCode == '0':
502	<	ErCode, msg = self.makeCopy( sbi, filetocopy , opt)
503	<	if self.debug : print 'Copy results for %s is %s'%( os.path.basename(filetocopy) ,ErCode)
493	>	try :
494	>	sbi = SBinterface( Source_SE, Destination_SE )
495	>	sbi_dest = SBinterface(Destination_SE)
496	>	sbi_source = SBinterface(Source_SE)
497	>	except ProtocolMismatch, ex:
498	>	msg  = "ERROR : Unable to create SBinterface with %s protocol"%protocol
499	>	msg += str(ex)
500	>	for filetocopy in list_file:
501	>	results.update( self.updateReport(filetocopy, '-1', msg))
502	>	return results
503	>
504	>
505	>	## loop over the complete list of files
506	>	for filetocopy in list_file:
507	>	if self.debug : print '\tStart real copy for %s'%filetocopy
508	>	try :
509	>	ErCode, msg = self.checkFileExist( sbi_source, sbi_dest, filetocopy, options )
510	>	except Exception, ex:
511	>	ErCode = '60307'
512	>	msg = str(ex)
513	>	if ErCode == '0':
514	>	ErCode, msg = self.makeCopy( sbi, filetocopy , options, protocol,sbi_dest )
515	>	if (ErCode == '0') and (backup == 'yes'):
516	>	ErCode = '60308'
517	>	if self.debug : print '\tCopy results for %s is %s'%( os.path.basename(filetocopy), ErCode)
518		results.update( self.updateReport(filetocopy, ErCode, msg))
519		return results
222	–
223	–	def updateReport(self, file, erCode, reason, lfn='', se='' ):
224	–	"""
225	–	Update the final stage out infos
226	–	"""
227	–	jobStageInfo={}
228	–	jobStageInfo['erCode']=erCode
229	–	jobStageInfo['reason']=reason
230	–	jobStageInfo['lfn']=lfn
231	–	jobStageInfo['se']=se
520
233	–	report = { file : jobStageInfo}
234	–	return report
235	–
236	–	def finalReport( self , results, middleware ):
237	–	"""
238	–	It should return a clear list of LFNs for each SE where data are stored.
239	–	allow "crab -copyLocal" or better "crab -copyOutput". TO_DO.
240	–	"""
241	–	if middleware:
242	–	outFile = open('cmscpReport.sh',"a")
243	–	cmscp_exit_status = 0
244	–	txt = ''
245	–	for file, dict in results.iteritems():
246	–	if dict['lfn']=='':
247	–	lfn = '$LFNBaseName/'+os.path.basename(file)
248	–	se  = '$SE'
249	–	else:
250	–	lfn = dict['lfn']+os.pat.basename(file)
251	–	se = dict['se']
252	–	#dict['lfn'] # to be implemented
253	–	txt +=  'echo "Report for File: '+file+'"\n'
254	–	txt +=  'echo "LFN: '+lfn+'"\n'
255	–	txt +=  'echo "StorageElement: '+se+'"\n'
256	–	txt += 'echo "StageOutExitStatusReason ='+dict['reason']+'" | tee -a $RUNTIME_AREA/$repo\n'
257	–	txt += 'echo "StageOutSE = '+se+'" >> $RUNTIME_AREA/$repo\n'
258	–	if dict['erCode'] != '0':
259	–	cmscp_exit_status = dict['erCode']
260	–	txt += '\n'
261	–	txt += 'export StageOutExitStatus='+str(cmscp_exit_status)+'\n'
262	–	txt +=  'echo "StageOutExitStatus = '+str(cmscp_exit_status)+'" | tee -a $RUNTIME_AREA/$repo\n'
263	–	outFile.write(str(txt))
264	–	outFile.close()
265	–	else:
266	–	for file, code in results.iteritems():
267	–	print 'error code = %s for file %s'%(code,file)
268	–	return
521
522		def storageInterface( self, endpoint, protocol ):
523		"""
524	<	Create the storage interface.
524	>	Create the storage interface.
525		"""
526	+	if self.debug : print 'storageInterface():\n'
527		try:
528		interface = SElement( FullPath(endpoint), protocol )
529	<	except Exception, ex:
530	<	msg = ''
531	<	if self.debug : msg = str(ex)+'\n'
532	<	msg += "ERROR : Unable to create interface with %s protocol\n"%protocol
280	<	print msg
529	>	except ProtocolUnknown, ex:
530	>	msg  = "ERROR : Unable to create interface with %s protocol"%protocol
531	>	msg += str(ex)
532	>	raise Exception(msg)
533
534		return interface
535
284	–	def checkDir(self, Destination_SE, protocol):
285	–	'''
286	–	ToBeImplemented NEEDED for castor
287	–	'''
288	–	return
289	–
536		def createDir(self, Destination_SE, protocol):
537		"""
538	<	Create remote dir for gsiftp/rfio REALLY TEMPORARY
539	<	this should be transparent at SE API level.
538	>	Create remote dir for gsiftp REALLY TEMPORARY
539	>	this should be transparent at SE API level.
540		"""
541	<	ErCode = '0'
542	<	msg_1 = ''
541	>	if self.debug : print 'createDir():\n'
542	>	msg = ''
543		try:
544		action = SBinterface( Destination_SE )
545		action.createDir()
546	<	if self.debug: print "The directory has been created using protocol %s\n"%protocol
547	<	except Exception, ex:
548	<	msg = ''
549	<	if self.debug : msg = str(ex)+'\n'
550	<	msg_1 = "ERROR: problem with the directory creation using %s protocol \n"%protocol
551	<	msg += msg_1
552	<	ErCode = '60316'
553	<	#print msg
554	<
555	<	return ErCode, msg_1
546	>	if self.debug: print "\tThe directory has been created using protocol %s"%protocol
547	>	except TransferException, ex:
548	>	msg  = "ERROR: problem with the directory creation using %s protocol "%protocol
549	>	msg += str(ex)
550	>	if self.debug :
551	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
552	>	dbgmsg += '\t'+str(ex.output)+'\n'
553	>	print dbgmsg
554	>	raise Exception(msg)
555	>	except OperationException, ex:
556	>	msg  = "ERROR: problem with the directory creation using %s protocol "%protocol
557	>	msg += str(ex)
558	>	if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n'
559	>	raise Exception(msg)
560	>	except MissingDestination, ex:
561	>	msg  = "ERROR: problem with the directory creation using %s protocol "%protocol
562	>	msg += str(ex)
563	>	if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n'
564	>	raise Exception(msg)
565	>	except AlreadyExistsException, ex:
566	>	if self.debug: print "\tThe directory already exist"
567	>	pass
568	>	except Exception, ex:
569	>	msg = "ERROR %s %s" %(str(ex), str(ex.detail))
570	>	if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n'
571	>	raise Exception(msg)
572	>	return msg
573
574	<	def checkFileExist(self, sbi, filetocopy):
574	>	def checkFileExist( self, sbi_source, sbi_dest, filetocopy, option ):
575		"""
576	<	Check if file to copy already exist
577	<	"""
578	<	try:
579	<	check = sbi.checkExists(filetocopy)
317	<	except Exception, ex:
318	<	msg = ''
319	<	if self.debug : msg = str(ex)+'\n'
320	<	msg += "ERROR: problem with check File Exist using %s protocol \n"%protocol
321	<	# print msg
576	>	Check both if source file exist AND
577	>	if destination file ALREADY exist.
578	>	"""
579	>	if self.debug : print 'checkFileExist():\n'
580		ErCode = '0'
581		msg = ''
582	<	if check :
583	<	ErCode = '60303'
584	<	msg = "file %s already exist"%filetocopy
585	<	print msg
582	>	f_tocopy=filetocopy
583	>	if self.source_prot != 'local':f_tocopy = os.path.basename(filetocopy)
584	>	try:
585	>	checkSource = sbi_source.checkExists( f_tocopy , opt=option, tout = self.subprocesstimeout['exists'] )
586	>	if self.debug : print '\tCheck for local file %s existance executed \n'%f_tocopy
587	>	except OperationException, ex:
588	>	msg  ='ERROR: problems checking source file %s existance'%filetocopy
589	>	msg += str(ex)
590	>	if self.debug :
591	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
592	>	dbgmsg += '\t'+str(ex.output)+'\n'
593	>	print dbgmsg
594	>	raise Exception(msg)
595	>	except WrongOption, ex:
596	>	msg  ='ERROR: problems checking source file %s existance'%filetocopy
597	>	msg += str(ex)
598	>	if self.debug :
599	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
600	>	dbgmsg += '\t'+str(ex.output)+'\n'
601	>	print dbgmsg
602	>	raise Exception(msg)
603	>	except MissingDestination, ex:
604	>	msg  ='ERROR: problems checking source file %s existance'%filetocopy
605	>	msg += str(ex)
606	>	if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n'
607	>	raise Exception(msg)
608	>	## when the client commands are not found (wrong env or really missing)
609	>	except MissingCommand, ex:
610	>	ErCode = '10041'
611	>	msg = "ERROR %s %s" %(str(ex), str(ex.detail))
612	>	return ErCode, msg
613	>	if not checkSource :
614	>	ErCode = '60302'
615	>	msg = "ERROR file %s does not exist"%os.path.basename(filetocopy)
616	>	return ErCode, msg
617	>	f_tocopy=os.path.basename(filetocopy)
618	>	try:
619	>	check = sbi_dest.checkExists( f_tocopy, opt=option, tout = self.subprocesstimeout['exists'] )
620	>	if self.debug : print '\tCheck for remote file %s existance executed \n'%f_tocopy
621	>	if self.debug : print '\twith exit code = %s \n'%check
622	>	except OperationException, ex:
623	>	msg  = 'ERROR: problems checking if file %s already exist'%filetocopy
624	>	msg += str(ex)
625	>	if self.debug :
626	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
627	>	dbgmsg += '\t'+str(ex.output)+'\n'
628	>	print dbgmsg
629	>	raise Exception(msg)
630	>	except WrongOption, ex:
631	>	msg  = 'ERROR problems checking if file % already exists'%filetocopy
632	>	msg += str(ex)
633	>	if self.debug :
634	>	msg += '\t'+msg+'\n\t'+str(ex.detail)+'\n'
635	>	msg += '\t'+str(ex.output)+'\n'
636	>	raise Exception(msg)
637	>	except MissingDestination, ex:
638	>	msg  ='ERROR problems checking if destination file % exists'%filetocopy
639	>	msg += str(ex)
640	>	if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n'
641	>	raise Exception(msg)
642	>	## when the client commands are not found (wrong env or really missing)
643	>	except MissingCommand, ex:
644	>	ErCode = '10041'
645	>	msg = "ERROR %s %s" %(str(ex), str(ex.detail))
646	>	return ErCode, msg
647	>	if check :
648	>	ErCode = '60303'
649	>	msg = "file %s already exist"%os.path.basename(filetocopy)
650
651	<	return ErCode,msg
651	>	return ErCode, msg
652
653	<	def makeCopy(self, sbi, filetocopy, opt ):
653	>	def makeCopy(self, sbi, filetocopy, option, protocol, sbi_dest ):
654		"""
655	<	call the copy API.
655	>	call the copy API.
656		"""
657	<	path = os.path.dirname(filetocopy)
657	>	if self.debug : print 'makeCopy():\n'
658	>	path = os.path.dirname(filetocopy)
659		file_name =  os.path.basename(filetocopy)
660		source_file = filetocopy
661		dest_file = file_name ## to be improved supporting changing file name  TODO
662	<	if self.source == '' and path == '':
662	>	if self.params['source'] == '' and path == '':
663		source_file = os.path.abspath(filetocopy)
664	<	elif self.destination =='':
665	<	dest_file = os.path.join(os.getcwd(),file_name)
666	<	elif self.source != '' and self.destination != '' :
667	<	source_file = file_name
664	>	elif self.params['destination'] =='':
665	>	destDir = self.params.get('destinationDir',os.getcwd())
666	>	dest_file = os.path.join(destDir,file_name)
667	>	elif self.params['source'] != '' and self.params['destination'] != '' :
668	>	source_file = file_name
669	>
670		ErCode = '0'
671		msg = ''
672	<
672	>
673	>	copy_option = option
674	>	if  self.params['option'].find('space_token')>=0:
675	>	space_token=self.params['option'].split('=')[1]
676	>	if protocol == 'srmv2': copy_option = '%s -space_token=%s'%(option,space_token)
677	>	if protocol == 'srm-lcg': copy_option = '%s -S %s'%(option,space_token)
678		try:
679	<	pippo = sbi.copy( source_file , dest_file , opt)
680	<	if self.protocol == 'srm' : self.checkSize( sbi, filetocopy )
681	<	except Exception, ex:
682	<	msg = ''
683	<	if self.debug : msg = str(ex)+'\n'
684	<	msg = "Problem copying %s file with %s command"%( filetocopy, protocol )
679	>	sbi.copy( source_file , dest_file , opt = copy_option, tout = self.subprocesstimeout['copy'])
680	>	except TransferException, ex:
681	>	msg  = "Problem copying %s file" % filetocopy
682	>	msg += str(ex)
683	>	if self.debug :
684	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
685	>	dbgmsg += '\t'+str(ex.output)+'\n'
686	>	print dbgmsg
687		ErCode = '60307'
688	<	#print msg
689	<
688	>	except WrongOption, ex:
689	>	msg  = "Problem copying %s file" % filetocopy
690	>	msg += str(ex)
691	>	if self.debug :
692	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
693	>	dbgmsg += '\t'+str(ex.output)+'\n'
694	>	print dbgmsg
695	>	### FEDE added for savannah 97460###
696	>	ErCode = '60307'
697	>	####################################
698	>	except SizeZeroException, ex:
699	>	msg  = "Problem copying %s file" % filetocopy
700	>	msg += str(ex)
701	>	if self.debug :
702	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
703	>	dbgmsg += '\t'+str(ex.output)+'\n'
704	>	print dbgmsg
705	>	ErCode = '60307'
706	>	## when the client commands are not found (wrong env or really missing)
707	>	except MissingCommand, ex:
708	>	ErCode = '10041'
709	>	msg  = "Problem copying %s file" % filetocopy
710	>	msg += str(ex)
711	>	if self.debug :
712	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
713	>	dbgmsg += '\t'+str(ex.output)+'\n'
714	>	print dbgmsg
715	>	except AuthorizationException, ex:
716	>	ErCode = '60307'
717	>	msg  = "Problem copying %s file" % filetocopy
718	>	msg += str(ex)
719	>	if self.debug :
720	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
721	>	dbgmsg += '\t'+str(ex.output)+'\n'
722	>	print dbgmsg
723	>	except SEAPITimeout, ex:
724	>	ErCode = '60317'
725	>	msg  = "Problem copying %s file" % filetocopy
726	>	msg += str(ex)
727	>	if self.debug :
728	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
729	>	dbgmsg += '\t'+str(ex.output)+'\n'
730	>	print dbgmsg
731	>
732	>	if ErCode == '0' and protocol.find('srmv') == 0:
733	>	remote_file_size = -1
734	>	local_file_size = os.path.getsize( source_file )
735	>	try:
736	>	remote_file_size = sbi_dest.getSize( dest_file, opt=option, tout = self.subprocesstimeout['size'] )
737	>	if self.debug : print '\t Check of remote size succeded for file %s\n'%dest_file
738	>	except TransferException, ex:
739	>	msg  = "Problem checking the size of %s file" % filetocopy
740	>	msg += str(ex)
741	>	if self.debug :
742	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
743	>	dbgmsg += '\t'+str(ex.output)+'\n'
744	>	print dbgmsg
745	>	ErCode = '60307'
746	>	except WrongOption, ex:
747	>	msg  = "Problem checking the size of %s file" % filetocopy
748	>	msg += str(ex)
749	>	if self.debug :
750	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
751	>	dbgmsg += '\t'+str(ex.output)+'\n'
752	>	print dbgmsg
753	>	ErCode = '60307'
754	>	if local_file_size != remote_file_size:
755	>	msg = "File size dosn't match: local size = %s ; remote size = %s " % (local_file_size, remote_file_size)
756	>	ErCode = '60307'
757	>
758	>	if ErCode != '0':
759	>	try :
760	>	self.removeFile( sbi_dest, dest_file, option )
761	>	except Exception, ex:
762	>	msg += '\n'+str(ex)
763		return ErCode, msg
764	<
765	<	'''
766	<	def checkSize()
764	>
765	>	def removeFile( self, sbi_dest, filetocopy, option ):
766	>	"""
767	>	"""
768	>	if self.debug : print 'removeFile():\n'
769	>	f_tocopy=filetocopy
770	>	if self.dest_prot != 'local':f_tocopy = os.path.basename(filetocopy)
771	>	try:
772	>	sbi_dest.delete( f_tocopy, opt=option, tout = self.subprocesstimeout['delete'] )
773	>	if self.debug : '\t deletion of file %s succeeded\n'%str(filetocopy)
774	>	except OperationException, ex:
775	>	msg  ='ERROR: problems removing partially staged file %s'%filetocopy
776	>	msg += str(ex)
777	>	if self.debug :
778	>	dbgmsg  = '\t'+msg+'\n\t'+str(ex.detail)+'\n'
779	>	dbgmsg += '\t'+str(ex.output)+'\n'
780	>	print dbgmsg
781	>	raise Exception(msg)
782	>
783	>	return
784	>
785	>	def updateReport(self, file, erCode, reason):
786		"""
787	<	Using srm needed a check of the ouptut file size.
787	>	Update the final stage out infos
788		"""
789	<
790	<	echo "--> remoteSize = $remoteSize"
791	<	## for local file
792	<	localSize=$(stat -c%s "$path_out_file")
793	<	echo "-->  localSize = $localSize"
794	<	if [ $localSize != $remoteSize ]; then
795	<	echo "Local fileSize $localSize does not match remote fileSize $remoteSize"
796	<	echo "Copy failed: removing remote file $destination"
797	<	srmrm $destination
798	<	cmscp_exit_status=60307
799	<
800	<
801	<	echo "Problem copying $path_out_file to $destination with srmcp command"
802	<	StageOutExitStatusReason='remote and local file dimension not match'
803	<	echo "StageOutReport = `cat ./srmcp.report`"
804	<	'''
805	<	def backup(self):
806	<	"""
807	<	Check infos from TFC using existing api obtaining:
808	<	1)destination
809	<	2)protocol
789	>	jobStageInfo={}
790	>	jobStageInfo['erCode']=erCode
791	>	jobStageInfo['reason']=reason
792	>	if not self.params['for_lfn']: self.params['for_lfn']=''
793	>	if not self.params['se_name']: self.params['se_name']=''
794	>	if not self.hostname: self.hostname=''
795	>	if (erCode != '0') and (erCode != '60308'):
796	>	jobStageInfo['for_lfn']='/copy_problem/'
797	>	else:
798	>	jobStageInfo['for_lfn']=self.params['for_lfn']
799	>	jobStageInfo['se_name']=self.params['se_name']
800	>	jobStageInfo['endpoint']=self.hostname
801	>	### ADDING SURLFORGRID FOR COPYDATA
802	>	if not self.params['surl_for_grid']: self.params['surl_for_grid']=''
803	>	jobStageInfo['surl_for_grid']=self.params['surl_for_grid']
804	>	#####
805	>	report = { file : jobStageInfo}
806	>	return report
807	>
808	>	def finalReport( self , results ):
809	>	"""
810	>	It a list of LFNs for each SE where data are stored.
811	>	allow "crab -copyLocal" or better "crab -copyOutput". TO_DO.
812		"""
813	+
814	+	outFile = 'cmscpReport.sh'
815	+	if os.getenv("RUNTIME_AREA"):
816	+	#print "RUNTIME_AREA = ", os.getenv("RUNTIME_AREA")
817	+	outFile = "%s/cmscpReport.sh"%os.getenv("RUNTIME_AREA")
818	+	#print "--->>> outFile = ", outFile
819	+	fp = open(outFile, "w")
820	+
821	+	cmscp_exit_status = 0
822	+	txt = '#!/bin/bash\n'
823	+	for file, dict in results.iteritems():
824	+	reason = str(dict['reason'])
825	+	if str(reason).find("'") > -1:
826	+	reason = " ".join(reason.split("'"))
827	+	reason="'%s'"%reason
828	+	if file:
829	+	if dict['for_lfn']=='':
830	+	lfn = '${LFNBaseName}'+os.path.basename(file)
831	+	se  = '$SE'
832	+	LFNBaseName = '$LFNBaseName'
833	+	else:
834	+	lfn = dict['for_lfn']+os.path.basename(file)
835	+	se = dict['se_name']
836	+	LFNBaseName = os.path.dirname(lfn)
837	+	if (LFNBaseName[-1] != '/'):
838	+	LFNBaseName = LFNBaseName + '/'
839	+
840	+
841	+	txt += 'echo "Report for File: '+file+'"\n'
842	+	txt += 'echo "LFN: '+lfn+'"\n'
843	+	txt += 'echo "StorageElement: '+se+'"\n'
844	+	txt += 'echo "StageOutExitStatusReason = %s" | tee -a $RUNTIME_AREA/$repo\n'%reason
845	+	txt += 'echo "StageOutSE = '+se+'" >> $RUNTIME_AREA/$repo\n'
846	+
847	+
848	+	if dict['erCode'] != '0':
849	+	cmscp_exit_status = dict['erCode']
850	+	else:
851	+	txt += 'echo "StageOutExitStatusReason = %s" | tee -a $RUNTIME_AREA/$repo\n'%reason
852	+	cmscp_exit_status = dict['erCode']
853	+	txt += '\n'
854	+	txt += 'export StageOutExitStatus='+str(cmscp_exit_status)+'\n'
855	+	txt += 'echo "StageOutExitStatus = '+str(cmscp_exit_status)+'" | tee -a $RUNTIME_AREA/$repo\n'
856	+	fp.write(str(txt))
857	+	fp.close()
858	+	if self.debug:
859	+	print '--- reading cmscpReport.sh: \n'
860	+	lp = open(outFile, "r")
861	+	content = lp.read()
862	+	lp.close()
863	+	print "    content = ", content
864	+	print '--- end reading cmscpReport.sh'
865		return
866
389	–	def usage(self):
867
868	<	msg="""
869	<	required parameters:
870	<	--source        :: REMOTE           :
871	<	--destination   :: REMOTE           :
872	<	--debug             :
873	<	--inFile :: absPath : or name NOT RELATIVE PATH
874	<	--outFIle :: onlyNAME : NOT YET SUPPORTED
875	<
876	<	optional parameters
877	<	"""
878	<	return msg
868	>	def usage():
869	>
870	>	msg="""
871	>	cmscp:
872	>	safe copy of local file  to/from remote SE via lcg_cp/srmcp,
873	>	including success checking  version also for CAF using rfcp command to copy the output to SE
874	>
875	>	accepted parameters:
876	>	source           =
877	>	destination      =
878	>	inputFileList    =
879	>	outputFileList   =
880	>	protocol         =
881	>	option           =
882	>	middleware       =
883	>	srm_version      =
884	>	destinationDir   =
885	>	lfn=             =
886	>	local_stage      =  activate stage fall back
887	>	debug            =  activate verbose print out
888	>	help             =  print on line man and exit
889	>
890	>	mandatory:
891	>	* "source" and/or "destination" must always be defined
892	>	* either "middleware" or "protocol" must always be defined
893	>	* "inputFileList" must always be defined
894	>	* if "local_stage" = 1 also  "lfn" must be defined
895	>	"""
896	>	print msg
897	>
898	>	return
899	>
900	>	def HelpOptions(opts=[]):
901	>	"""
902	>	Check otps, print help if needed
903	>	prepare dict = { opt : value }
904	>	"""
905	>	dict_args = {}
906	>	if len(opts):
907	>	for opt, arg in opts:
908	>	dict_args[opt.split('--')[1]] = arg
909	>	if opt in ('-h','-help','--help') :
910	>	usage()
911	>	sys.exit(0)
912	>	return dict_args
913	>	else:
914	>	usage()
915	>	sys.exit(0)
916
917		if __name__ == '__main__' :
918	+
919	+	import getopt
920	+
921	+	allowedOpt = ["source=", "destination=", "inputFileList=", "outputFileList=", \
922	+	"protocol=","option=", "middleware=", "srm_version=", \
923	+	"destinationDir=", "for_lfn=", "local_stage", "debug", "help", "se_name="]
924	+	try:
925	+	opts, args = getopt.getopt( sys.argv[1:], "", allowedOpt )
926	+	except getopt.GetoptError, err:
927	+	print err
928	+	HelpOptions()
929	+	sys.exit(2)
930	+
931	+	dictArgs = HelpOptions(opts)
932		try:
933	<	cmscp_ = cmscp(sys.argv[1:])
933	>	cmscp_ = cmscp(dictArgs)
934		cmscp_.run()
935	<	except:
936	<	pass
935	>	except Exception, ex :
936	>	print str(ex)
937

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines