1 |
< |
##!/usr/bin/env python |
2 |
< |
|
1 |
> |
#!/usr/bin/env python |
2 |
|
import sys, os |
3 |
|
from ProdCommon.Storage.SEAPI.SElement import SElement, FullPath |
4 |
|
from ProdCommon.Storage.SEAPI.SBinterface import * |
9 |
|
def __init__(self, args): |
10 |
|
""" |
11 |
|
cmscp |
12 |
< |
|
14 |
< |
safe copy of local file in current directory to remote SE via lcg_cp/srmcp, |
12 |
> |
safe copy of local file to/from remote SE via lcg_cp/srmcp, |
13 |
|
including success checking version also for CAF using rfcp command to copy the output to SE |
14 |
|
input: |
15 |
|
$1 middleware (CAF, LSF, LCG, OSG) |
17 |
|
$3 if needed: file name (the output file name) |
18 |
|
$5 remote SE (complete endpoint) |
19 |
|
$6 srm version |
20 |
+ |
--lfn $LFNBaseName |
21 |
|
output: |
22 |
|
return 0 if all ok |
23 |
|
return 60307 if srmcp failed |
26 |
|
|
27 |
|
#set default |
28 |
|
self.params = {"source":'', "destination":'','destinationDir':'', "inputFileList":'', "outputFileList":'', \ |
29 |
< |
"protocol":'', "option":'', "middleware":'', "srm_version":'srmv2'} |
29 |
> |
"protocol":'', "option":'', "middleware":'', "srm_version":'srmv2', "lfn":'' } |
30 |
|
self.debug = 0 |
31 |
< |
|
31 |
> |
self.local_stage = 0 |
32 |
|
self.params.update( args ) |
33 |
|
|
34 |
|
return |
39 |
|
""" |
40 |
|
if 'help' in self.params.keys(): HelpOptions() |
41 |
|
if 'debug' in self.params.keys(): self.debug = 1 |
42 |
+ |
if 'local_stage' in self.params.keys(): self.local_stage = 1 |
43 |
|
|
44 |
|
# source and dest cannot be undefined at same time |
45 |
|
if not self.params['source'] and not self.params['destination'] : |
59 |
|
file_to_copy.append(self.params['inputFileList']) |
60 |
|
self.params['inputFileList'] = file_to_copy |
61 |
|
|
62 |
+ |
if not self.params['lfn'] and self.local_stage == 1 : HelpOptions() |
63 |
+ |
|
64 |
|
## TO DO: |
65 |
|
#### add check for outFiles |
66 |
|
#### add map {'inFileNAME':'outFileNAME'} to change out name |
82 |
|
results = self.copy(self.params['inputFileList'], self.params['protocol'], self.params['option'] ) |
83 |
|
return results |
84 |
|
|
85 |
+ |
def checkLcgUtils( self ): |
86 |
+ |
""" |
87 |
+ |
_checkLcgUtils_ |
88 |
+ |
check the lcg-utils version and report |
89 |
+ |
""" |
90 |
+ |
import commands |
91 |
+ |
cmd = "lcg-cp --version | grep lcg_util" |
92 |
+ |
status, output = commands.getstatusoutput( cmd ) |
93 |
+ |
num_ver = -1 |
94 |
+ |
if output.find("not found") == -1 or status == 0: |
95 |
+ |
temp = output.split("-") |
96 |
+ |
version = "" |
97 |
+ |
if len(temp) >= 2: |
98 |
+ |
version = output.split("-")[1] |
99 |
+ |
temp = version.split(".") |
100 |
+ |
if len(temp) >= 1: |
101 |
+ |
num_ver = int(temp[0])*10 |
102 |
+ |
num_ver += int(temp[1]) |
103 |
+ |
return num_ver |
104 |
+ |
|
105 |
|
def setProtocol( self, middleware ): |
106 |
|
""" |
107 |
|
define the allowed potocols based on $middlware |
108 |
|
which depend on scheduler |
109 |
|
""" |
110 |
|
# default To be used with "middleware" |
111 |
+ |
if self.debug: |
112 |
+ |
print 'setProtocol() :\n' |
113 |
+ |
print '\tmiddleware = %s utils \n'%middleware |
114 |
+ |
|
115 |
|
lcgOpt={'srmv1':'-b -D srmv1 -t 2400 --verbose', |
116 |
|
'srmv2':'-b -D srmv2 -t 2400 --verbose'} |
117 |
+ |
if self.checkLcgUtils() >= 17: |
118 |
+ |
lcgOpt={'srmv1':'-b -D srmv1 --srm-timeout 2400 --sendreceive-timeout 240 --connect-timeout 240 --verbose', |
119 |
+ |
'srmv2':'-b -D srmv2 --srm-timeout 2400 --sendreceive-timeout 240 --connect-timeout 240 --verbose'} |
120 |
+ |
|
121 |
|
srmOpt={'srmv1':' -report ./srmcp.report -retry_timeout 480000 -retry_num 3 -streams_num=1 ', |
122 |
|
'srmv2':' -report=./srmcp.report -retry_timeout=480000 -retry_num=3 '} |
123 |
|
rfioOpt='' |
124 |
|
|
125 |
|
supported_protocol = None |
126 |
< |
if middleware.lower() in ['osg','lcg','condor']: |
126 |
> |
if middleware.lower() in ['osg','lcg','condor','sge']: |
127 |
|
supported_protocol = [('srm-lcg',lcgOpt[self.params['srm_version']]),\ |
128 |
|
(self.params['srm_version'],srmOpt[self.params['srm_version']])] |
129 |
|
elif middleware.lower() in ['lsf','caf']: |
130 |
|
supported_protocol = [('rfio',rfioOpt)] |
131 |
+ |
elif middleware.lower() in ['arc']: |
132 |
+ |
supported_protocol = [('srmv2','-debug'),('srmv1','-debug')] |
133 |
|
else: |
134 |
|
## here we can add support for any kind of protocol, |
135 |
|
## maybe some local schedulers need something dedicated |
136 |
|
pass |
137 |
|
return supported_protocol |
138 |
|
|
139 |
< |
# def checkCopy(self, copy_results, list_files): |
139 |
> |
|
140 |
> |
def checkCopy (self, copy_results, len_list_files, prot, lfn='', se=''): |
141 |
> |
""" |
142 |
> |
Checks the status of copy and update result dictionary |
143 |
|
""" |
109 |
– |
#results={} |
144 |
|
list_retry = [] |
145 |
< |
list_existing = [] |
145 |
> |
list_retry_localSE = [] |
146 |
> |
list_not_existing = [] |
147 |
|
list_ok = [] |
148 |
< |
if copy_results.keys() == '': |
149 |
< |
self.results.update(copy_results) |
150 |
< |
else: |
151 |
< |
for file, dict in copy_results.iteritems(): |
152 |
< |
er_code = dict['erCode'] |
153 |
< |
if er_code == '0': |
154 |
< |
list_ok.append(file) |
155 |
< |
reason = 'Copy succedeed with %s utils'%prot |
156 |
< |
upDict = self.updateReport(file, er_code, reason) |
157 |
< |
copy_results.update(upDict) |
158 |
< |
elif er_code == '60303': list_existing.append( file ) |
159 |
< |
else: list_retry.append( file ) |
160 |
< |
results.update(copy_results) |
161 |
< |
if len(list_ok) != 0: |
162 |
< |
msg = 'Copy of %s succedeed with %s utils\n'%(str(list_ok),prot) |
163 |
< |
if self.debug : print msg |
164 |
< |
if len(list_ok) == len(list_files) : |
165 |
< |
msg = 'Copy of all files succedeed\n' |
166 |
< |
#break |
148 |
> |
|
149 |
> |
if self.debug: |
150 |
> |
print 'in checkCopy() :\n' |
151 |
> |
for file, dict in copy_results.iteritems(): |
152 |
> |
er_code = dict['erCode'] |
153 |
> |
if er_code == '0': |
154 |
> |
list_ok.append(file) |
155 |
> |
reason = 'Copy succedeed with %s utils'%prot |
156 |
> |
dict['reason'] = reason |
157 |
> |
elif er_code == '60302': |
158 |
> |
list_not_existing.append( file ) |
159 |
> |
elif er_code == '10041': |
160 |
> |
list_retry.append( file ) |
161 |
> |
## WHAT TO DO IN GENERAL FAILURE CONDITION |
162 |
> |
else: |
163 |
> |
list_retry_localSE.append( file ) |
164 |
> |
|
165 |
> |
if self.debug: |
166 |
> |
print "\t file %s \n"%file |
167 |
> |
print "\t dict['erCode'] %s \n"%dict['erCode'] |
168 |
> |
print "\t dict['reason'] %s \n"%dict['reason'] |
169 |
> |
|
170 |
> |
if (lfn != '') and (se != ''): |
171 |
> |
upDict = self.updateReport(file, er_code, dict['reason'], lfn, se) |
172 |
|
else: |
173 |
< |
if self.debug : print 'Copy of files %s failed using %s...\n'%(str(list_retry)+str(list_existing),prot) |
174 |
< |
#if len(list_retry): list_files = list_retry |
175 |
< |
return list_retry, results |
173 |
> |
upDict = self.updateReport(file, er_code, dict['reason']) |
174 |
> |
|
175 |
> |
copy_results.update(upDict) |
176 |
|
|
177 |
< |
""" |
177 |
> |
msg = '' |
178 |
> |
if len(list_ok) != 0: |
179 |
> |
msg += '\tCopy of %s succedeed with %s utils\n'%(str(list_ok),prot) |
180 |
> |
if len(list_ok) != len_list_files : |
181 |
> |
msg += '\tCopy of %s failed using %s for files \n'%(str(list_retry),prot) |
182 |
> |
msg += '\tCopy of %s failed using %s : files not found \n'%(str(list_not_existing),prot) |
183 |
> |
if self.debug : print msg |
184 |
> |
|
185 |
> |
return copy_results, list_ok, list_retry, list_retry_localSE |
186 |
> |
|
187 |
> |
def LocalCopy(self, list_retry, results): |
188 |
> |
""" |
189 |
> |
Tries the stage out to the CloseSE |
190 |
> |
""" |
191 |
> |
if self.debug: |
192 |
> |
print 'in LocalCopy() :\n' |
193 |
> |
print '\t list_retry %s utils \n'%list_retry |
194 |
> |
print '\t len(list_retry) %s \n'%len(list_retry) |
195 |
> |
|
196 |
> |
list_files = list_retry |
197 |
> |
self.params['inputFilesList']=list_files |
198 |
> |
|
199 |
> |
### copy backup |
200 |
> |
from ProdCommon.FwkJobRep.SiteLocalConfig import loadSiteLocalConfig |
201 |
> |
siteCfg = loadSiteLocalConfig() |
202 |
> |
seName = siteCfg.localStageOut.get("se-name", None) |
203 |
> |
catalog = siteCfg.localStageOut.get("catalog", None) |
204 |
> |
implName = siteCfg.localStageOut.get("command", None) |
205 |
> |
if (implName == 'srm'): |
206 |
> |
implName='srmv1' |
207 |
> |
self.params['srm_version']=implName |
208 |
> |
##### to be improved ############### |
209 |
> |
if (implName == 'rfcp'): |
210 |
> |
self.params['middleware']='lsf' |
211 |
> |
#################################### |
212 |
> |
|
213 |
> |
self.params['protocol']=implName |
214 |
> |
tfc = siteCfg.trivialFileCatalog() |
215 |
> |
|
216 |
> |
if self.debug: |
217 |
> |
print '\t siteCFG %s \n'%siteCfg |
218 |
> |
print '\t seName %s \n'%seName |
219 |
> |
print '\t catalog %s \n'%catalog |
220 |
> |
print "\t self.params['protocol'] %s \n"%self.params['protocol'] |
221 |
> |
print '\t tfc %s '%tfc |
222 |
> |
print "\t self.params['inputFilesList'] %s \n"%self.params['inputFilesList'] |
223 |
> |
|
224 |
> |
file_backup=[] |
225 |
> |
for input in self.params['inputFilesList']: |
226 |
> |
file = self.params['lfn'] + os.path.basename(input) |
227 |
> |
surl = tfc.matchLFN(tfc.preferredProtocol, file) |
228 |
> |
file_backup.append(surl) |
229 |
> |
if self.debug: |
230 |
> |
print '\t lfn %s \n'%self.params['lfn'] |
231 |
> |
print '\t file %s \n'%file |
232 |
> |
print '\t surl %s \n'%surl |
233 |
> |
|
234 |
> |
destination=os.path.dirname(file_backup[0]) |
235 |
> |
### FEDE added check for final / |
236 |
> |
if ( destination[-1] != '/' ) : destination = destination + '/' |
237 |
> |
##################################### |
238 |
> |
self.params['destination']=destination |
239 |
> |
|
240 |
> |
if self.debug: |
241 |
> |
print "\t self.params['destination']%s \n"%self.params['destination'] |
242 |
> |
print "\t self.params['protocol'] %s \n"%self.params['protocol'] |
243 |
> |
print "\t self.params['option']%s \n"%self.params['option'] |
244 |
> |
|
245 |
> |
for prot, opt in self.setProtocol( self.params['middleware'] ): |
246 |
> |
if self.debug: print '\tIn LocalCopy trying the stage out with %s utils \n'%prot |
247 |
> |
localCopy_results = self.copy( self.params['inputFileList'], prot, opt ) |
248 |
> |
if localCopy_results.keys() == [''] or localCopy_results.keys() == '' : |
249 |
> |
results.update(localCopy_results) |
250 |
> |
else: |
251 |
> |
localCopy_results, list_ok, list_retry, list_retry_localSE = self.checkCopy(localCopy_results, len(list_files), prot, self.params['lfn'], seName) |
252 |
> |
results.update(localCopy_results) |
253 |
> |
if len(list_ok) == len(list_files) : |
254 |
> |
break |
255 |
> |
if len(list_retry): |
256 |
> |
list_files = list_retry |
257 |
> |
else: break |
258 |
> |
if self.debug: |
259 |
> |
print "\t localCopy_results = %s \n"%localCopy_results |
260 |
> |
|
261 |
> |
return results |
262 |
> |
|
263 |
|
def stager( self, middleware, list_files ): |
264 |
|
""" |
265 |
|
Implement the logic for remote stage out |
266 |
|
""" |
267 |
|
|
268 |
< |
if self.debug: print 'stager() :\n' |
268 |
> |
if self.debug: |
269 |
> |
print 'stager() :\n' |
270 |
> |
print '\tmiddleware %s\n'%middleware |
271 |
> |
print '\tlist_files %s\n'%list_files |
272 |
> |
|
273 |
|
results={} |
274 |
|
for prot, opt in self.setProtocol( middleware ): |
275 |
|
if self.debug: print '\tTrying the stage out with %s utils \n'%prot |
276 |
|
copy_results = self.copy( list_files, prot, opt ) |
148 |
– |
######## to define a new function checkCopy ################ |
149 |
– |
#list_retry, self.results = self.checkCopy(copy_results, list_files) |
150 |
– |
#def checkCopy (self, copy_results): |
151 |
– |
# """ |
152 |
– |
# """ |
153 |
– |
# results={} |
154 |
– |
list_retry = [] |
155 |
– |
list_existing = [] |
156 |
– |
list_ok = [] |
277 |
|
if copy_results.keys() == [''] or copy_results.keys() == '' : |
278 |
|
results.update(copy_results) |
279 |
|
else: |
280 |
< |
for file, dict in copy_results.iteritems(): |
161 |
< |
er_code = dict['erCode'] |
162 |
< |
if er_code == '0': |
163 |
< |
list_ok.append(file) |
164 |
< |
reason = 'Copy succedeed with %s utils'%prot |
165 |
< |
upDict = self.updateReport(file, er_code, reason) |
166 |
< |
copy_results.update(upDict) |
167 |
< |
elif er_code in ['60303','60302']: list_existing.append( file ) |
168 |
< |
else: list_retry.append( file ) |
280 |
> |
copy_results, list_ok, list_retry, list_retry_localSE = self.checkCopy(copy_results, len(list_files), prot) |
281 |
|
results.update(copy_results) |
170 |
– |
msg = '' |
171 |
– |
if len(list_ok) != 0: |
172 |
– |
msg += '\tCopy of %s succedeed with %s utils\n'%(str(list_ok),prot) |
282 |
|
if len(list_ok) == len(list_files) : |
283 |
|
break |
284 |
< |
else: |
285 |
< |
if self.debug: msg += '\tCopy of %s failed using %s...\n'%(str(list_retry)+str(list_existing),prot) |
286 |
< |
if len(list_retry): list_files = list_retry |
287 |
< |
else: break |
288 |
< |
if self.debug : print msg |
289 |
< |
""" |
290 |
< |
if len(list_retry): |
291 |
< |
list_files = list_retry |
292 |
< |
#def backupCopy(list_retry) |
293 |
< |
print "in backup" |
185 |
< |
self.params['inputFilesList']=list_files |
186 |
< |
### copy backup |
187 |
< |
from ProdCommon.FwkJobRep.SiteLocalConfig import loadSiteLocalConfig |
188 |
< |
siteCfg = loadSiteLocalConfig() |
189 |
< |
#print siteCfg |
190 |
< |
seName = siteCfg.localStageOut.get("se-name", None) |
191 |
< |
#print "seName = ", seName |
192 |
< |
self.params['destination']=seName |
193 |
< |
#catalog = siteCfg.localStageOut.get("catalog", None) |
194 |
< |
#print "catalog = ", catalog |
195 |
< |
implName = siteCfg.localStageOut.get("command", None) |
196 |
< |
print "implName = ", implName |
197 |
< |
if (implName == 'srm'): |
198 |
< |
implName='srmv2' |
199 |
< |
self.params['protocol']=implName |
200 |
< |
tfc = siteCfg.trivialFileCatalog() |
201 |
< |
#print "tfc = ", tfc |
202 |
< |
print " self.params['inputFilesList'] = ", self.params['inputFilesList'] |
203 |
< |
file_backup=[] |
204 |
< |
for input in self.params['inputFilesList']: |
205 |
< |
### to add the correct lfn, passed as argument of cmscp function (--lfn xxxx) |
206 |
< |
file = '/store/'+input |
207 |
< |
pfn = tfc.matchLFN(tfc.preferredProtocol, file) |
208 |
< |
print "pfn = ", pfn |
209 |
< |
file_backup.append(pfn) |
210 |
< |
self.params['inputFilesList'] = file_backup |
211 |
< |
print "#########################################" |
212 |
< |
print "self.params['inputFilesList'] = ", self.params['inputFilesList'] |
213 |
< |
print "self.params['protocol'] = ", self.params['protocol'] |
214 |
< |
print "self.params['option'] = ", self.params['option'] |
215 |
< |
self.copy(self.params['inputFilesList'], self.params['protocol'], self.params['option']) |
216 |
< |
print "#########################################" |
217 |
< |
###list_retry, self.results = checkCopy(copy_results) |
218 |
< |
#check is something fails and created related dict |
219 |
< |
# backup = self.analyzeResults(results) |
220 |
< |
# if backup : |
221 |
< |
# msg = 'WARNING: backup logic is under implementation\n' |
222 |
< |
# #backupDict = self.backup() |
223 |
< |
# ### NOTE: IT MUST RETURN a DICT contains also LFN and SE Name |
224 |
< |
# results.update(backupDict) |
225 |
< |
# print msg |
226 |
< |
""" |
227 |
< |
#### TODO Daniele |
228 |
< |
#check is something fails and created related dict |
229 |
< |
# backup = self.analyzeResults(results) |
230 |
< |
|
231 |
< |
# if backup : |
232 |
< |
# msg = 'WARNING: backup logic is under implementation\n' |
233 |
< |
# #backupDict = self.backup() |
234 |
< |
# ### NOTE: IT MUST RETURN a DICT contains also LFN and SE Name |
235 |
< |
# results.update(backupDict) |
236 |
< |
# print msg |
284 |
> |
if len(list_retry): |
285 |
> |
list_files = list_retry |
286 |
> |
else: break |
287 |
> |
|
288 |
> |
if self.local_stage: |
289 |
> |
if len(list_retry_localSE): |
290 |
> |
results = self.LocalCopy(list_retry_localSE, results) |
291 |
> |
|
292 |
> |
if self.debug: |
293 |
> |
print "\t results %s \n"%results |
294 |
|
return results |
295 |
|
|
296 |
|
def initializeApi(self, protocol ): |
330 |
|
if Destination_SE.protocol in ['gridftp','rfio','srmv2']: |
331 |
|
try: |
332 |
|
self.createDir( Destination_SE, Destination_SE.protocol ) |
333 |
< |
except Exception, ex: |
333 |
> |
except OperationException, ex: |
334 |
|
return self.updateReport('', '60316', str(ex)) |
335 |
+ |
## when the client commands are not found (wrong env or really missing) |
336 |
+ |
except MissingCommand, ex: |
337 |
+ |
msg = "ERROR %s %s" %(str(ex), str(ex.detail)) |
338 |
+ |
return self.updateReport('', '10041', msg) |
339 |
|
|
340 |
|
## prepare for real copy ## |
341 |
|
try : |
354 |
|
try : |
355 |
|
ErCode, msg = self.checkFileExist( sbi_source, sbi_dest, filetocopy, options ) |
356 |
|
except Exception, ex: |
357 |
< |
ErCode = -1 |
357 |
> |
ErCode = '60307' |
358 |
|
msg = str(ex) |
359 |
|
if ErCode == '0': |
360 |
|
ErCode, msg = self.makeCopy( sbi, filetocopy , options, protocol,sbi_dest ) |
445 |
|
msg += str(ex) |
446 |
|
if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n' |
447 |
|
raise Exception(msg) |
448 |
+ |
## when the client commands are not found (wrong env or really missing) |
449 |
+ |
except MissingCommand, ex: |
450 |
+ |
ErCode = '10041' |
451 |
+ |
msg = "ERROR %s %s" %(str(ex), str(ex.detail)) |
452 |
+ |
return ErCode, msg |
453 |
|
if not checkSource : |
454 |
|
ErCode = '60302' |
455 |
|
msg = "ERROR file %s do not exist"%os.path.basename(filetocopy) |
479 |
|
msg += str(ex) |
480 |
|
if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n' |
481 |
|
raise Exception(msg) |
482 |
+ |
## when the client commands are not found (wrong env or really missing) |
483 |
+ |
except MissingCommand, ex: |
484 |
+ |
ErCode = '10041' |
485 |
+ |
msg = "ERROR %s %s" %(str(ex), str(ex.detail)) |
486 |
+ |
return ErCode, msg |
487 |
|
if check : |
488 |
|
ErCode = '60303' |
489 |
|
msg = "file %s already exist"%os.path.basename(filetocopy) |
510 |
|
ErCode = '0' |
511 |
|
msg = '' |
512 |
|
|
513 |
+ |
if self.params['option'].find('space_token'): |
514 |
+ |
space_tocken=self.params['option'].split('=')[1] |
515 |
+ |
if protocol == 'srmv2': option = '%s -space_tocken=%s'%(option,space_tocken) |
516 |
+ |
if protocol == 'srm-lcg': option = '%s -S %s'%(option,space_tocken) |
517 |
|
try: |
518 |
|
sbi.copy( source_file , dest_file , opt = option) |
519 |
|
except TransferException, ex: |
530 |
|
if self.debug : |
531 |
|
dbgmsg = '\t'+msg+'\n\t'+str(ex.detail)+'\n' |
532 |
|
dbgmsg += '\t'+str(ex.output)+'\n' |
533 |
< |
print dbsmsg |
533 |
> |
print dbgmsg |
534 |
> |
except SizeZeroException, ex: |
535 |
> |
msg = "Problem copying %s file" % filetocopy |
536 |
> |
msg += str(ex) |
537 |
> |
if self.debug : |
538 |
> |
dbgmsg = '\t'+msg+'\n\t'+str(ex.detail)+'\n' |
539 |
> |
dbgmsg += '\t'+str(ex.output)+'\n' |
540 |
> |
print dbgmsg |
541 |
|
ErCode = '60307' |
542 |
+ |
## when the client commands are not found (wrong env or really missing) |
543 |
+ |
except MissingCommand, ex: |
544 |
+ |
ErCode = '10041' |
545 |
+ |
msg = "Problem copying %s file" % filetocopy |
546 |
+ |
msg += str(ex) |
547 |
+ |
if self.debug : |
548 |
+ |
dbgmsg = '\t'+msg+'\n\t'+str(ex.detail)+'\n' |
549 |
+ |
dbgmsg += '\t'+str(ex.output)+'\n' |
550 |
+ |
print dbgmsg |
551 |
+ |
except AuthorizationException, ex: |
552 |
+ |
ErCode = '60307' |
553 |
+ |
msg = "Problem copying %s file" % filetocopy |
554 |
+ |
msg += str(ex) |
555 |
+ |
if self.debug : |
556 |
+ |
dbgmsg = '\t'+msg+'\n\t'+str(ex.detail)+'\n' |
557 |
+ |
dbgmsg += '\t'+str(ex.output)+'\n' |
558 |
+ |
print dbgmsg |
559 |
|
if ErCode == '0' and protocol.find('srmv') == 0: |
560 |
|
remote_file_size = -1 |
561 |
|
local_file_size = os.path.getsize( source_file ) |
609 |
|
|
610 |
|
return |
611 |
|
|
513 |
– |
def backup(self): |
514 |
– |
""" |
515 |
– |
Check infos from TFC using existing api obtaining: |
516 |
– |
1)destination |
517 |
– |
2)protocol |
518 |
– |
""" |
519 |
– |
return |
520 |
– |
|
612 |
|
def updateReport(self, file, erCode, reason, lfn='', se='' ): |
613 |
|
""" |
614 |
|
Update the final stage out infos |
631 |
|
cmscp_exit_status = 0 |
632 |
|
txt = '' |
633 |
|
for file, dict in results.iteritems(): |
634 |
+ |
reason = str(dict['reason']) |
635 |
+ |
if str(reason).find("'") > -1: |
636 |
+ |
reason = " ".join(reason.split("'")) |
637 |
+ |
reason="'%s'"%reason |
638 |
|
if file: |
639 |
|
if dict['lfn']=='': |
640 |
|
lfn = '$LFNBaseName/'+os.path.basename(file) |
643 |
|
lfn = dict['lfn']+os.path.basename(file) |
644 |
|
se = dict['se'] |
645 |
|
#dict['lfn'] # to be implemented |
646 |
< |
txt += 'echo "Report for File: '+file+'"\n' |
647 |
< |
txt += 'echo "LFN: '+lfn+'"\n' |
648 |
< |
txt += 'echo "StorageElement: '+se+'"\n' |
649 |
< |
txt += 'echo "StageOutExitStatusReason ='+dict['reason']+'" | tee -a $RUNTIME_AREA/$repo\n' |
646 |
> |
txt += 'echo "Report for File: '+file+'"\n' |
647 |
> |
txt += 'echo "LFN: '+lfn+'"\n' |
648 |
> |
txt += 'echo "StorageElement: '+se+'"\n' |
649 |
> |
txt += 'echo "StageOutExitStatusReason = %s" | tee -a $RUNTIME_AREA/$repo\n'%reason |
650 |
|
txt += 'echo "StageOutSE = '+se+'" >> $RUNTIME_AREA/$repo\n' |
651 |
+ |
#txt += 'export LFNBaseName='+lfn+'\n' |
652 |
+ |
txt += 'export SE='+se+'\n' |
653 |
+ |
### FEDE per CopyData #### |
654 |
+ |
|
655 |
+ |
txt += 'export endpoint='+self.params['destination']+'\n' |
656 |
+ |
|
657 |
|
if dict['erCode'] != '0': |
658 |
|
cmscp_exit_status = dict['erCode'] |
558 |
– |
cmscp_exit_status = dict['erCode'] |
659 |
|
else: |
660 |
< |
txt += 'echo "StageOutExitStatusReason ='+dict['reason']+'" | tee -a $RUNTIME_AREA/$repo\n' |
660 |
> |
txt += 'echo "StageOutExitStatusReason = %s" | tee -a $RUNTIME_AREA/$repo\n'%reason |
661 |
|
cmscp_exit_status = dict['erCode'] |
662 |
|
cmscp_exit_status = dict['erCode'] |
663 |
|
txt += '\n' |
671 |
|
def usage(): |
672 |
|
|
673 |
|
msg=""" |
674 |
< |
required parameters: |
675 |
< |
--source :: REMOTE : |
676 |
< |
--destination :: REMOTE : |
577 |
< |
--debug : |
578 |
< |
--inFile :: absPath : or name NOT RELATIVE PATH |
579 |
< |
--outFIle :: onlyNAME : NOT YET SUPPORTED |
674 |
> |
cmscp: |
675 |
> |
safe copy of local file to/from remote SE via lcg_cp/srmcp, |
676 |
> |
including success checking version also for CAF using rfcp command to copy the output to SE |
677 |
|
|
678 |
< |
optional parameters |
678 |
> |
accepted parameters: |
679 |
> |
source = |
680 |
> |
destination = |
681 |
> |
inputFileList = |
682 |
> |
outputFileList = |
683 |
> |
protocol = |
684 |
> |
option = |
685 |
> |
middleware = |
686 |
> |
srm_version = |
687 |
> |
destinationDir = |
688 |
> |
lfn= = |
689 |
> |
local_stage = activate stage fall back |
690 |
> |
debug = activate verbose print out |
691 |
> |
help = print on line man and exit |
692 |
> |
|
693 |
> |
mandatory: |
694 |
> |
* "source" and/or "destination" must always be defined |
695 |
> |
* either "middleware" or "protocol" must always be defined |
696 |
> |
* "inputFileList" must always be defined |
697 |
> |
* if "local_stage" = 1 also "lfn" must be defined |
698 |
|
""" |
699 |
|
print msg |
700 |
|
|
723 |
|
|
724 |
|
allowedOpt = ["source=", "destination=", "inputFileList=", "outputFileList=", \ |
725 |
|
"protocol=","option=", "middleware=", "srm_version=", \ |
726 |
< |
"destinationDir=","debug", "help"] |
726 |
> |
"destinationDir=", "lfn=", "local_stage", "debug", "help"] |
727 |
|
try: |
728 |
|
opts, args = getopt.getopt( sys.argv[1:], "", allowedOpt ) |
729 |
|
except getopt.GetoptError, err: |