| 18 |
|
"owner","parent_job", "reason","resubmitted","rsl","seed",\ |
| 19 |
|
"stateEnterTime","stateEnterTimes","subjob_failed", \ |
| 20 |
|
"user tags" , "status" , "status_code","hierarchy"] |
| 21 |
+ |
|
| 22 |
|
return |
| 23 |
|
|
| 24 |
|
def configure(self, cfg_params): |
| 60 |
|
try: self.VO = cfg_params['EDG.virtual_organization'] |
| 61 |
|
except KeyError: self.VO = 'cms' |
| 62 |
|
|
| 63 |
+ |
try: self.copy_input_data = cfg_params["USER.copy_input_data"] |
| 64 |
+ |
except KeyError: self.copy_input_data = 0 |
| 65 |
+ |
|
| 66 |
|
try: self.return_data = cfg_params['USER.return_data'] |
| 67 |
|
except KeyError: self.return_data = 0 |
| 68 |
|
|
| 168 |
|
try: self.schedulerName = cfg_params['CRAB.scheduler'] |
| 169 |
|
except KeyError: self.scheduler = '' |
| 170 |
|
|
| 171 |
+ |
try: self.dontCheckProxy=cfg_params["EDG.dont_check_proxy"] |
| 172 |
+ |
except KeyError: self.dontCheckProxy = 0 |
| 173 |
+ |
|
| 174 |
|
return |
| 175 |
|
|
| 176 |
|
|
| 219 |
|
for i in range(len(ce_white_list)): |
| 220 |
|
if i == 0: |
| 221 |
|
if (req == ' '): |
| 222 |
< |
req = req + '((RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))' |
| 222 |
> |
req = req + '((RegExp("' + string.strip(ce_white_list[i]) + '", other.GlueCEUniqueId))' |
| 223 |
|
else: |
| 224 |
< |
req = req + ' && ((RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))' |
| 224 |
> |
req = req + ' && ((RegExp("' + string.strip(ce_white_list[i]) + '", other.GlueCEUniqueId))' |
| 225 |
|
pass |
| 226 |
|
else: |
| 227 |
< |
req = req + ' || (RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))' |
| 227 |
> |
req = req + ' || (RegExp("' + string.strip(ce_white_list[i]) + '", other.GlueCEUniqueId))' |
| 228 |
|
req = req + ')' |
| 229 |
|
|
| 230 |
|
if self.EDG_ce_black_list: |
| 231 |
|
ce_black_list = string.split(self.EDG_ce_black_list,',') |
| 232 |
|
for ce in ce_black_list: |
| 233 |
|
if (req == ' '): |
| 234 |
< |
req = req + '(!RegExp("' + ce + '", other.GlueCEUniqueId))' |
| 234 |
> |
req = req + '(!RegExp("' + string.strip(ce) + '", other.GlueCEUniqueId))' |
| 235 |
|
else: |
| 236 |
< |
req = req + ' && (!RegExp("' + ce + '", other.GlueCEUniqueId))' |
| 236 |
> |
req = req + ' && (!RegExp("' + string.strip(ce) + '", other.GlueCEUniqueId))' |
| 237 |
|
pass |
| 238 |
|
if self.EDG_clock_time: |
| 239 |
|
if (req == ' '): |
| 248 |
|
req = req + ' && other.GlueCEPolicyMaxCPUTime>='+self.EDG_cpu_time |
| 249 |
|
|
| 250 |
|
for i in range(len(first)): # Add loop DS |
| 251 |
+ |
groupReq = req |
| 252 |
|
self.param='sched_param_'+str(i)+'.clad' |
| 253 |
|
param_file = open(common.work_space.shareDir()+'/'+self.param, 'w') |
| 254 |
|
|
| 255 |
|
itr4=self.findSites_(first[i]) |
| 256 |
|
for arg in itr4: |
| 257 |
< |
req = req + ' && anyMatch(other.storage.CloseSEs, ('+str(arg)+'))' |
| 258 |
< |
param_file.write('Requirements = '+req +';\n') |
| 257 |
> |
groupReq = groupReq + ' && anyMatch(other.storage.CloseSEs, ('+str(arg)+'))' |
| 258 |
> |
param_file.write('Requirements = '+groupReq +';\n') |
| 259 |
|
|
| 260 |
|
if (self.rb_param_file != ''): |
| 261 |
|
param_file.write(self.rb_param_file) |
| 401 |
|
Copy input data from SE to WN |
| 402 |
|
""" |
| 403 |
|
txt = '' |
| 404 |
+ |
if not self.copy_input_data: return txt |
| 405 |
|
|
| 406 |
|
## OLI_Daniele deactivate for OSG (wait for LCG UI installed on OSG) |
| 407 |
|
txt += 'if [ $middleware == OSG ]; then\n' |
| 497 |
|
txt += ' echo "StageOutExitStatus = 198" | tee -a $RUNTIME_AREA/$repo\n' |
| 498 |
|
txt += ' echo "StageOutExitStatusReason = $exitstring" | tee -a $RUNTIME_AREA/$repo\n' |
| 499 |
|
txt += ' echo "srmcp and lcg-cp and failed!"\n' |
| 500 |
+ |
txt += ' SE=""\n' |
| 501 |
+ |
txt += ' echo "SE = $SE"\n' |
| 502 |
+ |
txt += ' SE_PATH=""\n' |
| 503 |
+ |
txt += ' echo "SE_PATH = $SE_PATH"\n' |
| 504 |
|
txt += ' else\n' |
| 505 |
|
txt += ' echo "StageOutSE = $SE" | tee -a $RUNTIME_AREA/$repo\n' |
| 506 |
|
txt += ' echo "StageOutCatalog = " | tee -a $RUNTIME_AREA/$repo\n' |
| 516 |
|
txt += ' echo "srmcp succeeded"\n' |
| 517 |
|
txt += ' fi\n' |
| 518 |
|
txt += ' done\n' |
| 519 |
+ |
txt += ' exit_status=$copy_exit_status\n' |
| 520 |
|
return txt |
| 521 |
|
|
| 522 |
|
def wsRegisterOutput(self): |
| 589 |
|
""" |
| 590 |
|
self.checkProxy() |
| 591 |
|
cmd = 'edg-job-get-logging-info -v 2 ' + id |
| 578 |
– |
#cmd_out = os.popen(cmd) |
| 592 |
|
cmd_out = runCommand(cmd) |
| 593 |
|
return cmd_out |
| 594 |
|
|
| 582 |
– |
def getExitStatus(self, id): |
| 583 |
– |
return self.getStatusAttribute_(id, 'exit_code') |
| 584 |
– |
|
| 585 |
– |
def queryStatus(self, id): |
| 586 |
– |
return self.getStatusAttribute_(id, 'status') |
| 587 |
– |
|
| 588 |
– |
def queryDest(self, id): |
| 589 |
– |
return self.getStatusAttribute_(id, 'destination') |
| 590 |
– |
|
| 591 |
– |
|
| 592 |
– |
def getStatusAttribute_(self, id, attr): |
| 593 |
– |
""" Query a status of the job with id """ |
| 594 |
– |
|
| 595 |
– |
self.checkProxy() |
| 596 |
– |
hstates = {} |
| 597 |
– |
Status = importName('edg_wl_userinterface_common_LbWrapper', 'Status') |
| 598 |
– |
# Bypass edg-job-status interfacing directly to C++ API |
| 599 |
– |
# Job attribute vector to retrieve status without edg-job-status |
| 600 |
– |
level = 0 |
| 601 |
– |
# Instance of the Status class provided by LB API |
| 602 |
– |
jobStat = Status() |
| 603 |
– |
st = 0 |
| 604 |
– |
#print id, level, attr, self.states.index(attr) |
| 605 |
– |
jobStat.getStatus(id, level) |
| 606 |
– |
#print jobStat.loadStatus(st) |
| 607 |
– |
err, apiMsg = jobStat.get_error() |
| 608 |
– |
if err: |
| 609 |
– |
common.logger.debug(5,'Error caught' + apiMsg) |
| 610 |
– |
return None |
| 611 |
– |
else: |
| 612 |
– |
for i in range(len(self.states)): |
| 613 |
– |
# Fill an hash table with all information retrieved from LB API |
| 614 |
– |
hstates[ self.states[i] ] = jobStat.loadStatus(st)[i] |
| 615 |
– |
#print i, jobStat.loadStatus(st)[i] |
| 616 |
– |
result = jobStat.loadStatus(st)[self.states.index(attr)] |
| 617 |
– |
#print str(result) |
| 618 |
– |
return result |
| 619 |
– |
|
| 595 |
|
def queryDetailedStatus(self, id): |
| 596 |
|
""" Query a detailed status of the job with id """ |
| 597 |
|
cmd = 'edg-job-status '+id |
| 614 |
|
return itr4 |
| 615 |
|
|
| 616 |
|
def createXMLSchScript(self, nj, argsList): |
| 642 |
– |
# def createXMLSchScript(self, nj): |
| 617 |
|
|
| 618 |
|
""" |
| 619 |
|
Create a XML-file for BOSS4. |
| 629 |
|
jbt = job.type() |
| 630 |
|
|
| 631 |
|
inp_sandbox = jbt.inputSandbox(index) |
| 632 |
< |
out_sandbox = jbt.outputSandbox(index) |
| 632 |
> |
#out_sandbox = jbt.outputSandbox(index) |
| 633 |
|
""" |
| 634 |
|
[end] FIX-ME |
| 635 |
|
""" |
| 645 |
|
dir = string.split(common.work_space.topDir(), '/') |
| 646 |
|
taskName = dir[len(dir)-2] |
| 647 |
|
|
| 674 |
– |
to_writeReq = '' |
| 648 |
|
to_write = '' |
| 649 |
|
|
| 650 |
|
req=' ' |
| 699 |
|
to_write = to_write + 'MyProxyServer = ""' + self.proxyServer + '""\n' |
| 700 |
|
to_write = to_write + 'VirtualOrganisation = ""' + self.VO + '""\n' |
| 701 |
|
|
| 702 |
< |
#TaskName |
| 702 |
> |
#TaskName |
| 703 |
|
dir = string.split(common.work_space.topDir(), '/') |
| 704 |
|
taskName = dir[len(dir)-2] |
| 705 |
|
|
| 706 |
|
xml.write(str(title)) |
| 707 |
< |
xml.write('<task name="' +str(taskName)+'">\n') |
| 707 |
> |
#xml.write('<task name="' +str(taskName)+'" sub_path="' +common.work_space.pathForTgz() + 'share/.boss_cache">\n') |
| 708 |
> |
|
| 709 |
> |
#xml.write('<task name="' +str(taskName)+ '" sub_path="' +common.work_space.pathForTgz() + 'share/.boss_cache"' + '" task_info="' + os.path.expandvars('X509_USER_PROXY') + '">\n') |
| 710 |
> |
xml.write('<task name="' +str(taskName)+ '" sub_path="' +common.work_space.pathForTgz() + 'share/.boss_cache"' + ' task_info="' + os.environ["X509_USER_PROXY"] + '">\n') |
| 711 |
|
xml.write(jt_string) |
| 712 |
|
|
| 713 |
|
if (to_write != ''): |
| 734 |
|
indy: here itr4 |
| 735 |
|
''' |
| 736 |
|
|
| 737 |
< |
|
| 738 |
< |
xml.write('<chain scheduler="'+str(self.schedulerName)+'">\n') |
| 737 |
> |
xml.write('<chain name="' +str(taskName)+'__ITR1_" scheduler="'+str(self.schedulerName)+'">\n') |
| 738 |
> |
# xml.write('<chain scheduler="'+str(self.schedulerName)+'">\n') |
| 739 |
|
xml.write(jt_string) |
| 740 |
|
|
| 741 |
|
#executable |
| 748 |
|
xml.write('<program>\n') |
| 749 |
|
xml.write('<exec> ' + os.path.basename(script) +' </exec>\n') |
| 750 |
|
xml.write(jt_string) |
| 751 |
< |
|
| 751 |
> |
|
| 752 |
|
xml.write('<args> <![CDATA[\n _ITR2_ \n]]> </args>\n') |
| 753 |
|
xml.write('<program_types> crabjob </program_types>\n') |
| 754 |
< |
inp_box = script + ',' |
| 754 |
> |
inp_box = common.work_space.pathForTgz() + 'job/' + jbt.scriptName + ',' |
| 755 |
|
|
| 756 |
|
if inp_sandbox != None: |
| 757 |
|
for fl in inp_sandbox: |
| 759 |
|
pass |
| 760 |
|
pass |
| 761 |
|
|
| 762 |
< |
inp_box = inp_box + os.path.abspath(os.environ['CRABDIR']+'/python/'+'report.py') + ',' +\ |
| 763 |
< |
os.path.abspath(os.environ['CRABDIR']+'/python/'+'DashboardAPI.py') + ','+\ |
| 764 |
< |
os.path.abspath(os.environ['CRABDIR']+'/python/'+'Logger.py') + ','+\ |
| 765 |
< |
os.path.abspath(os.environ['CRABDIR']+'/python/'+'ProcInfo.py') + ','+\ |
| 766 |
< |
os.path.abspath(os.environ['CRABDIR']+'/python/'+'apmon.py') + ','+\ |
| 767 |
< |
os.path.abspath(os.environ['CRABDIR']+'/python/'+'parseCrabFjr.py') |
| 792 |
< |
|
| 793 |
< |
if (not jbt.additional_inbox_files == []): |
| 794 |
< |
inp_box = inp_box + ',' |
| 795 |
< |
for addFile in jbt.additional_inbox_files: |
| 796 |
< |
addFile = os.path.abspath(addFile) |
| 797 |
< |
inp_box = inp_box+''+addFile+',' |
| 798 |
< |
pass |
| 762 |
> |
# if (not jbt.additional_inbox_files == []): |
| 763 |
> |
# inp_box = inp_box + ',' |
| 764 |
> |
# for addFile in jbt.additional_inbox_files: |
| 765 |
> |
# #addFile = os.path.abspath(addFile) |
| 766 |
> |
# inp_box = inp_box+''+addFile+',' |
| 767 |
> |
# pass |
| 768 |
|
|
| 769 |
|
if inp_box[-1] == ',' : inp_box = inp_box[:-1] |
| 770 |
|
inp_box = '<infiles> <![CDATA[\n' + inp_box + '\n]]> </infiles>\n' |
| 795 |
|
INDY |
| 796 |
|
something similar should be also done for infiles (if it makes sense!) |
| 797 |
|
""" |
| 798 |
+ |
# Stuff to be returned _always_ via sandbox |
| 799 |
+ |
for fl in jbt.output_file_sandbox: |
| 800 |
+ |
out_box = out_box + '' + jbt.numberFile_(fl, '_ITR1_') + ',' |
| 801 |
+ |
pass |
| 802 |
+ |
pass |
| 803 |
+ |
|
| 804 |
+ |
# via sandbox iif required return_data |
| 805 |
|
if int(self.return_data) == 1: |
| 806 |
|
for fl in jbt.output_file: |
| 807 |
|
out_box = out_box + '' + jbt.numberFile_(fl, '_ITR1_') + ',' |
| 830 |
|
Function to check the Globus proxy. |
| 831 |
|
""" |
| 832 |
|
if (self.proxyValid): return |
| 833 |
< |
timeleft = -999 |
| 833 |
> |
|
| 834 |
> |
### Just return if asked to do so |
| 835 |
> |
if (self.dontCheckProxy==1): |
| 836 |
> |
self.proxyValid=1 |
| 837 |
> |
return |
| 838 |
> |
|
| 839 |
|
minTimeLeft=10*3600 # in seconds |
| 840 |
|
|
| 841 |
|
minTimeLeftServer = 100 # in hours |
| 884 |
|
reTime = re.compile( r'timeleft: (\d+)' ) |
| 885 |
|
#print "<"+str(reTime.search( cmd_out ).group(1))+">" |
| 886 |
|
if reTime.match( cmd_out ): |
| 887 |
< |
time = reTime.search( line ).group(1) |
| 887 |
> |
time = reTime.search( cmd_out ).group(1) |
| 888 |
|
if time < minTimeLeftServer: |
| 889 |
|
renewProxy = 1 |
| 890 |
|
common.logger.message('No credential delegation will expire in '+time+' hours: renew it') |
