| 1 |
|
#!/usr/bin/env python |
| 2 |
|
import sys, os |
| 3 |
+ |
import time, random |
| 4 |
|
try: |
| 5 |
|
import json |
| 6 |
|
except: |
| 100 |
|
if self.debug: |
| 101 |
|
print 'in writeJsonFile() : \n' |
| 102 |
|
print "---->>>> in writeJsonFile results = ", results |
| 103 |
< |
fp = open('resultCopyFile', 'w') |
| 103 |
> |
jsonOut = "resultCopyFile" |
| 104 |
> |
if os.getenv("RUNTIME_AREA"): |
| 105 |
> |
jsonOut = "%s/resultCopyFile"%os.getenv("RUNTIME_AREA") |
| 106 |
> |
fp = open(jsonOut, 'w') |
| 107 |
|
json.dump(results, fp) |
| 108 |
|
fp.close() |
| 109 |
|
if self.debug: |
| 110 |
|
print ' reading resultCopyFile : \n' |
| 111 |
< |
lp = open('resultCopyFile', "r") |
| 111 |
> |
lp = open(jsonOut, "r") |
| 112 |
|
inputDict = json.load(lp) |
| 113 |
|
lp.close() |
| 114 |
|
print " inputDict = ", inputDict |
| 153 |
|
srmOpt={'srmv1':' -report ./srmcp.report -retry_timeout 480000 -retry_num 3 -streams_num=1 ', |
| 154 |
|
'srmv2':' -report=./srmcp.report -retry_timeout=480000 -retry_num=3 -storagetype=permanent '} |
| 155 |
|
rfioOpt='' |
| 156 |
+ |
#### FEDE FOR XROOTD ######### |
| 157 |
+ |
xrootdOpt='' |
| 158 |
+ |
############################# |
| 159 |
|
|
| 160 |
|
supported_protocol = None |
| 161 |
|
if middleware.lower() in ['osg','lcg','condor','sge']: |
| 162 |
< |
supported_protocol = [('srm-lcg',lcgOpt[self.params['srm_version']]),\ |
| 163 |
< |
(self.params['srm_version'],srmOpt[self.params['srm_version']])] |
| 164 |
< |
elif middleware.lower() in ['lsf','caf']: |
| 162 |
> |
supported_protocol = [('srm-lcg',lcgOpt[self.params['srm_version']])]#,\ |
| 163 |
> |
# (self.params['srm_version'],srmOpt[self.params['srm_version']])] |
| 164 |
> |
#elif middleware.lower() in ['lsf','caf']: |
| 165 |
> |
elif middleware.lower() in ['lsf']: |
| 166 |
|
supported_protocol = [('rfio',rfioOpt)] |
| 167 |
|
elif middleware.lower() in ['pbs']: |
| 168 |
|
supported_protocol = [('rfio',rfioOpt),('local','')] |
| 169 |
|
elif middleware.lower() in ['arc']: |
| 170 |
|
supported_protocol = [('srmv2','-debug'),('srmv1','-debug')] |
| 171 |
+ |
#### FEDE FOR XROOTD ########## |
| 172 |
+ |
elif middleware.lower() in ['caf']: |
| 173 |
+ |
if self.params['protocol']: |
| 174 |
+ |
supported_protocol = [(self.params['protocol'], '')] |
| 175 |
+ |
else: |
| 176 |
+ |
supported_protocol = [('rfio',rfioOpt)] |
| 177 |
+ |
####################################### |
| 178 |
|
else: |
| 179 |
|
## here we can add support for any kind of protocol, |
| 180 |
|
## maybe some local schedulers need something dedicated |
| 400 |
|
break |
| 401 |
|
if len(list_retry): |
| 402 |
|
list_files = list_retry |
| 403 |
+ |
#### FEDE added ramdom time before the retry copy with other protocol |
| 404 |
+ |
sec = 240 * random.random() |
| 405 |
+ |
sec = sec + 60 |
| 406 |
+ |
time.sleep(sec) |
| 407 |
|
else: break |
| 408 |
|
|
| 409 |
|
if self.local_stage: |
| 572 |
|
if self.source_prot != 'local':f_tocopy = os.path.basename(filetocopy) |
| 573 |
|
try: |
| 574 |
|
checkSource = sbi_source.checkExists( f_tocopy , opt=option, tout = self.subprocesstimeout['exists'] ) |
| 575 |
< |
if self.debug : print '\tCheck for local file %s exist succeded \n'%f_tocopy |
| 575 |
> |
if self.debug : print '\tCheck for local file %s existance executed \n'%f_tocopy |
| 576 |
|
except OperationException, ex: |
| 577 |
< |
msg ='ERROR: problems checkig if source file %s exist'%filetocopy |
| 577 |
> |
msg ='ERROR: problems checking source file %s existance'%filetocopy |
| 578 |
|
msg += str(ex) |
| 579 |
|
if self.debug : |
| 580 |
|
dbgmsg = '\t'+msg+'\n\t'+str(ex.detail)+'\n' |
| 582 |
|
print dbgmsg |
| 583 |
|
raise Exception(msg) |
| 584 |
|
except WrongOption, ex: |
| 585 |
< |
msg ='ERROR problems checkig if source file % exist'%filetocopy |
| 585 |
> |
msg ='ERROR: problems checking source file %s existance'%filetocopy |
| 586 |
|
msg += str(ex) |
| 587 |
|
if self.debug : |
| 588 |
|
dbgmsg = '\t'+msg+'\n\t'+str(ex.detail)+'\n' |
| 590 |
|
print dbgmsg |
| 591 |
|
raise Exception(msg) |
| 592 |
|
except MissingDestination, ex: |
| 593 |
< |
msg ='ERROR problems checkig if source file % exist'%filetocopy |
| 593 |
> |
msg ='ERROR: problems checking source file %s existance'%filetocopy |
| 594 |
|
msg += str(ex) |
| 595 |
|
if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n' |
| 596 |
|
raise Exception(msg) |
| 601 |
|
return ErCode, msg |
| 602 |
|
if not checkSource : |
| 603 |
|
ErCode = '60302' |
| 604 |
< |
msg = "ERROR file %s do not exist"%os.path.basename(filetocopy) |
| 604 |
> |
msg = "ERROR file %s does not exist"%os.path.basename(filetocopy) |
| 605 |
|
return ErCode, msg |
| 606 |
|
f_tocopy=os.path.basename(filetocopy) |
| 607 |
|
try: |
| 608 |
|
check = sbi_dest.checkExists( f_tocopy, opt=option, tout = self.subprocesstimeout['exists'] ) |
| 609 |
< |
if self.debug : print '\tCheck for remote file %s exist done \n'%f_tocopy |
| 609 |
> |
if self.debug : print '\tCheck for remote file %s existance executed \n'%f_tocopy |
| 610 |
|
if self.debug : print '\twith exit code = %s \n'%check |
| 611 |
|
except OperationException, ex: |
| 612 |
< |
msg = 'ERROR: problems checkig if file %s already exist'%filetocopy |
| 612 |
> |
msg = 'ERROR: problems checking if file %s already exist'%filetocopy |
| 613 |
|
msg += str(ex) |
| 614 |
|
if self.debug : |
| 615 |
|
dbgmsg = '\t'+msg+'\n\t'+str(ex.detail)+'\n' |
| 617 |
|
print dbgmsg |
| 618 |
|
raise Exception(msg) |
| 619 |
|
except WrongOption, ex: |
| 620 |
< |
msg = 'ERROR problems checkig if file % already exist'%filetocopy |
| 620 |
> |
msg = 'ERROR problems checking if file % already exists'%filetocopy |
| 621 |
|
msg += str(ex) |
| 622 |
|
if self.debug : |
| 623 |
|
msg += '\t'+msg+'\n\t'+str(ex.detail)+'\n' |
| 624 |
|
msg += '\t'+str(ex.output)+'\n' |
| 625 |
|
raise Exception(msg) |
| 626 |
|
except MissingDestination, ex: |
| 627 |
< |
msg ='ERROR problems checkig if source file % exist'%filetocopy |
| 627 |
> |
msg ='ERROR problems checking if destination file % exists'%filetocopy |
| 628 |
|
msg += str(ex) |
| 629 |
|
if self.debug : print '\t'+msg+'\n\t'+str(ex.detail)+'\n' |
| 630 |
|
raise Exception(msg) |
| 680 |
|
if self.debug : |
| 681 |
|
dbgmsg = '\t'+msg+'\n\t'+str(ex.detail)+'\n' |
| 682 |
|
dbgmsg += '\t'+str(ex.output)+'\n' |
| 683 |
< |
print dbgmsg |
| 683 |
> |
print dbgmsg |
| 684 |
> |
### FEDE added for savannah 97460### |
| 685 |
> |
ErCode = '60307' |
| 686 |
> |
#################################### |
| 687 |
|
except SizeZeroException, ex: |
| 688 |
|
msg = "Problem copying %s file" % filetocopy |
| 689 |
|
msg += str(ex) |
| 799 |
|
It a list of LFNs for each SE where data are stored. |
| 800 |
|
allow "crab -copyLocal" or better "crab -copyOutput". TO_DO. |
| 801 |
|
""" |
| 802 |
< |
outFile = open('cmscpReport.sh',"a") |
| 802 |
> |
|
| 803 |
> |
outFile = 'cmscpReport.sh' |
| 804 |
> |
if os.getenv("RUNTIME_AREA"): |
| 805 |
> |
#print "RUNTIME_AREA = ", os.getenv("RUNTIME_AREA") |
| 806 |
> |
outFile = "%s/cmscpReport.sh"%os.getenv("RUNTIME_AREA") |
| 807 |
> |
#print "--->>> outFile = ", outFile |
| 808 |
> |
fp = open(outFile, "w") |
| 809 |
> |
|
| 810 |
|
cmscp_exit_status = 0 |
| 811 |
< |
txt = '' |
| 811 |
> |
txt = '#!/bin/bash\n' |
| 812 |
|
for file, dict in results.iteritems(): |
| 813 |
|
reason = str(dict['reason']) |
| 814 |
|
if str(reason).find("'") > -1: |
| 842 |
|
txt += '\n' |
| 843 |
|
txt += 'export StageOutExitStatus='+str(cmscp_exit_status)+'\n' |
| 844 |
|
txt += 'echo "StageOutExitStatus = '+str(cmscp_exit_status)+'" | tee -a $RUNTIME_AREA/$repo\n' |
| 845 |
< |
outFile.write(str(txt)) |
| 846 |
< |
outFile.close() |
| 845 |
> |
fp.write(str(txt)) |
| 846 |
> |
fp.close() |
| 847 |
> |
if self.debug: |
| 848 |
> |
print '--- reading cmscpReport.sh: \n' |
| 849 |
> |
lp = open(outFile, "r") |
| 850 |
> |
content = lp.read() |
| 851 |
> |
lp.close() |
| 852 |
> |
print " content = ", content |
| 853 |
> |
print '--- end reading cmscpReport.sh' |
| 854 |
|
return |
| 855 |
|
|
| 856 |
|
|
