| 85 |
|
if not os.path.isdir(self.logDir) or not os.path.isdir(self.outDir): |
| 86 |
|
msg = ' Output or Log dir not found!! check '+self.logDir+' and '+self.outDir |
| 87 |
|
raise CrabException(msg) |
| 88 |
< |
else: |
| 89 |
< |
submission_id = common._db.queryRunJob('submission',self.list_id) |
| 90 |
< |
submission_id.sort() |
| 91 |
< |
submission_id.reverse() |
| 92 |
< |
max_id=submission_id[0] |
| 93 |
< |
if max_id > 1: self.moveOutput(max_id) |
| 88 |
> |
#else: |
| 89 |
> |
# submission_id = common._db.queryRunJob('submission',self.list_id) |
| 90 |
> |
# submission_id.sort() |
| 91 |
> |
# submission_id.reverse() |
| 92 |
> |
# max_id=submission_id[0] |
| 93 |
> |
# if max_id > 1: self.moveOutput(max_id) |
| 94 |
|
|
| 95 |
|
return |
| 96 |
|
|
| 110 |
|
listCode = [] |
| 111 |
|
job_id = [] |
| 112 |
|
|
| 113 |
< |
cwd = os.getcwd() |
| 114 |
< |
os.chdir( self.outDir ) |
| 113 |
> |
#cwd = os.getcwd() |
| 114 |
> |
#os.chdir( self.outDir ) |
| 115 |
|
success_ret = 0 |
| 116 |
|
for id in self.list_id: |
| 117 |
|
file = 'out_files_'+ str(id)+'.tgz' |
| 118 |
< |
if os.path.exists(file): |
| 119 |
< |
cmd = 'tar zxvf '+file |
| 118 |
> |
if os.path.exists(self.outDir + file): |
| 119 |
> |
self.submission_id = common._db.queryRunJob('submission',id) |
| 120 |
> |
self.max_id=max(self.submission_id) |
| 121 |
> |
if self.max_id > 1: |
| 122 |
> |
for f in os.listdir(self.outDir): |
| 123 |
> |
if (f.find(str(id)) != -1 ) and (f != file): |
| 124 |
> |
self.moveOutput(id, self.max_id, self.outDir, f) |
| 125 |
> |
if self.log==1: |
| 126 |
> |
for f in os.listdir(self.logDir): |
| 127 |
> |
if f.find(str(id)) != -1: |
| 128 |
> |
self.moveOutput(id, self.max_id, self.logDir, f) |
| 129 |
> |
cmd = 'tar zxvf ' + self.outDir + file + ' ' + '-C ' + self.outDir |
| 130 |
|
cmd_out = runCommand(cmd) |
| 131 |
< |
cmd_2 ='rm out_files_'+ str(id)+'.tgz' |
| 131 |
> |
cmd_2 ='rm ' + self.outDir + 'out_files_'+ str(id)+'.tgz' |
| 132 |
> |
#cmd_2 ='rm out_files_'+ str(id)+'.tgz' |
| 133 |
|
cmd_out2 = runCommand(cmd_2) |
| 134 |
|
msg = 'Results of Jobs # '+str(id)+' are in '+self.outDir |
| 135 |
|
common.logger.message(msg) |
| 138 |
|
common.logger.debug(1,msg) |
| 139 |
|
continue |
| 140 |
|
input = 'crab_fjr_' + str(id) + '.xml' |
| 141 |
< |
if os.path.exists(input): |
| 142 |
< |
codeValue = self.parseFinalReport(input) |
| 141 |
> |
if os.path.exists(self.outDir + input): |
| 142 |
> |
codeValue = self.parseFinalReport(self.outDir + input) |
| 143 |
|
job_id.append(id) |
| 144 |
|
listCode.append(codeValue) |
| 145 |
|
else: |
| 146 |
|
msg = "Problems with "+str(input)+". File not available.\n" |
| 147 |
|
common.logger.message(msg) |
| 148 |
|
success_ret +=1 |
| 149 |
< |
os.chdir( cwd ) |
| 149 |
> |
#os.chdir( cwd ) |
| 150 |
|
common._db.updateRunJob_(job_id , listCode) |
| 151 |
|
|
| 152 |
|
if self.logDir != self.outDir: |
| 208 |
|
|
| 209 |
|
return codeValue |
| 210 |
|
|
| 211 |
< |
def moveOutput(self,max_id): |
| 211 |
> |
def moveOutput(self,id, max_id,path,file): |
| 212 |
|
""" |
| 213 |
|
Move output of job already retrieved |
| 214 |
|
into the correct backup directory |
| 215 |
|
""" |
| 216 |
< |
sub_id = common._db.queryRunJob('submission',self.list_id) |
| 217 |
< |
Dist_sub_id = list(set(sub_id)) |
| 218 |
< |
|
| 219 |
< |
OutDir_Base=self.outDir+'Submission_' |
| 220 |
< |
for i in range(1,max_id): |
| 210 |
< |
if not os.path.isdir( OutDir_Base + str(i) + '/'): |
| 211 |
< |
cmd=('mkdir '+ OutDir_Base + str(i) + '/ >& /dev/null') |
| 216 |
> |
Dir_Base=path +'Submission_' |
| 217 |
> |
|
| 218 |
> |
for i in range(1, max_id): |
| 219 |
> |
if not os.path.isdir( Dir_Base + str(i) + '/'): |
| 220 |
> |
cmd=('mkdir '+ Dir_Base + str(i) + '/ >& /dev/null') |
| 221 |
|
cmd_out = runCommand(cmd) |
| 222 |
|
common.logger.write(str(cmd_out)) |
| 223 |
|
common.logger.debug(3,str(cmd_out)) |
| 224 |
< |
if self.log == 1: |
| 225 |
< |
LogDir_Base=self.logDir+'Submission_' |
| 226 |
< |
if not os.path.isdir( LogDir_Base + str(i) + '/'): |
| 227 |
< |
cmd=('mkdir '+ LogDir_Base + str(i) + '/ >& /dev/null') |
| 228 |
< |
cmd_out = runCommand(cmd) |
| 229 |
< |
common.logger.write(str(cmd_out)) |
| 230 |
< |
common.logger.debug(3,str(cmd_out)) |
| 231 |
< |
for i in range(len(self.list_id)): |
| 232 |
< |
id = self.list_id[i] |
| 233 |
< |
if sub_id[i] > 1 : |
| 234 |
< |
cmd='mv '+self.outDir+'*_'+str(self.list_id[i])+'.* ' + OutDir_Base + str(sub_id[i]-1) + '/ >& /dev/null' |
| 226 |
< |
#else: |
| 227 |
< |
# cmd='mv '+self.outDir+'*_'+str(self.list_id[i])+'.* ' + OutDir_Base + str(sub_id[i]) + '/ >& /dev/null' |
| 228 |
< |
try: |
| 229 |
< |
cmd_out = runCommand(cmd) |
| 230 |
< |
common.logger.write(cmd_out) |
| 231 |
< |
common.logger.debug(3,cmd_out) |
| 232 |
< |
except: |
| 233 |
< |
msg = 'no output to move for job '+str(id) |
| 234 |
< |
common.logger.write(msg) |
| 235 |
< |
common.logger.debug(3,msg) |
| 236 |
< |
pass |
| 237 |
< |
if self.log == 1: |
| 238 |
< |
cmdlog='mv '+self.logDir+'*_'+str(self.list_id[i])+'.* ' + LogDir_Base + str(sub_id[i]-1) + '/ >& /dev/null' |
| 239 |
< |
try: |
| 240 |
< |
cmd_out = runCommand(cmdlog) |
| 241 |
< |
common.logger.write(cmd_out) |
| 242 |
< |
common.logger.debug(3,cmd_out) |
| 243 |
< |
except: |
| 244 |
< |
msg = 'no output to move for job '+str(id) |
| 245 |
< |
common.logger.write(msg) |
| 246 |
< |
common.logger.debug(3,msg) |
| 247 |
< |
pass |
| 224 |
> |
cmd='mv '+ path + file + ' ' + Dir_Base + str(max_id -1) + '/ >& /dev/null' |
| 225 |
> |
|
| 226 |
> |
try: |
| 227 |
> |
cmd_out = runCommand(cmd) |
| 228 |
> |
common.logger.write(cmd_out) |
| 229 |
> |
common.logger.debug(3,cmd_out) |
| 230 |
> |
except: |
| 231 |
> |
msg = 'no output to move for job '+str(id) |
| 232 |
> |
common.logger.write(msg) |
| 233 |
> |
common.logger.debug(3,msg) |
| 234 |
> |
pass |
| 235 |
|
return |
