ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/SchedulerLocal.py
Revision: 1.50
Committed: Mon Mar 9 15:56:26 2009 UTC (16 years, 1 month ago) by ewv
Content type: text/x-python
Branch: MAIN
CVS Tags: CRAB_2_6_0_pre2, CRAB_2_6_0_pre1, CRAB_2_5_1, CRAB_2_5_1_pre4, CRAB_2_5_1_pre3, CRAB_2_5_1_pre2, CRAB_2_5_1_pre1, CRAB_2_5_0, CRAB_2_5_0_pre7
Changes since 1.49: +10 -10 lines
Log Message: 
Fix CondorG problems with uniqueID, rename function to follow scheme

File Contents

# User Rev Content
1 slacapra 1.1 from Scheduler import Scheduler
2     from crab_exceptions import *
3     from crab_logger import Logger
4 spiga 1.47 from crab_util import getLocalDomain
5 slacapra 1.1 import common
6 spiga 1.27 from PhEDExDatasvcInfo import PhEDExDatasvcInfo
7 slacapra 1.1
8     import os,string
9    
10 slacapra 1.2 # Base class for all local scheduler
11 slacapra 1.1
12     class SchedulerLocal(Scheduler) :
13    
14     def configure(self, cfg_params):
15 spiga 1.47 self.environment_unique_identifier = None
16 spiga 1.27 self.cfg_params = cfg_params
17 spiga 1.46 Scheduler.configure(self,cfg_params)
18 slacapra 1.1 self.jobtypeName = cfg_params['CRAB.jobtype']
19    
20     name=string.upper(self.name())
21     self.queue = cfg_params.get(name+'.queue',None)
22    
23     self.res = cfg_params.get(name+'.resource',None)
24    
25     if (cfg_params.has_key(self.name()+'.env_id')): self.environment_unique_identifier = cfg_params[self.name()+'.env_id']
26     ## is this ok?
27 spiga 1.25 localDomainName = getLocalDomain(self)
28 slacapra 1.16 if not cfg_params.has_key('EDG.se_white_list'):
29     cfg_params['EDG.se_white_list']=localDomainName
30     common.logger.message("Your domain name is "+str(localDomainName)+": only local dataset will be considered")
31     else:
32     common.logger.message("Your se_white_list is set to "+str(cfg_params['EDG.se_white_list'])+": only local dataset will be considered")
33 slacapra 1.1 return
34    
35     def userName(self):
36     """ return the user name """
37 slacapra 1.5 import pwd,getpass
38     tmp=pwd.getpwnam(getpass.getuser())[4]
39     return "/CN="+tmp.strip()
40 slacapra 1.1
41 ewv 1.50 def envUniqueID(self):
42 spiga 1.47 return
43    
44 slacapra 1.1 def wsSetupEnvironment(self):
45     """
46     Returns part of a job script which does scheduler-specific work.
47     """
48 spiga 1.47 taskId = common._db.queryTask('name')
49 slacapra 1.1 if not self.environment_unique_identifier:
50 spiga 1.47 try :
51 ewv 1.50 self.environment_unique_identifier = self.envUniqueID()
52     except :
53 spiga 1.47 raise CrabException('environment_unique_identifier not set')
54 slacapra 1.12 index = int(common._db.nJobs())
55     job = common.job_list[index-1]
56     jbt = job.type()
57 ewv 1.20 # start with wrapper timing
58 spiga 1.18 txt = 'export TIME_WRAP_INI=`date +%s` \n'
59 farinafa 1.22 txt += 'export TIME_STAGEOUT=-2 \n\n'
60 spiga 1.17
61 spiga 1.19 txt += '# '+self.name()+' specific stuff\n'
62 slacapra 1.1 txt += '# strip arguments\n'
63     txt += 'echo "strip arguments"\n'
64     txt += 'args=("$@")\n'
65     txt += 'nargs=$#\n'
66     txt += 'shift $nargs\n'
67     txt += "# job number (first parameter for job wrapper)\n"
68 ewv 1.9 txt += "NJob=${args[0]}; export NJob\n"
69 slacapra 1.1
70 slacapra 1.12 txt += "out_files=out_files_${NJob}; export out_files\n"
71     txt += "echo $out_files\n"
72     txt += jbt.outList()
73    
74 slacapra 1.6 txt += 'SyncGridJobId=`echo '+self.environment_unique_identifier+'`\n'
75     txt += 'MonitorJobID=`echo ${NJob}_${SyncGridJobId}`\n'
76 spiga 1.38 txt += 'MonitorID=`echo ' + taskId + '`\n'
77 slacapra 1.1
78     txt += 'echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
79     txt += 'echo "SyncGridJobId=`echo $SyncGridJobId`" | tee -a $RUNTIME_AREA/$repo \n'
80     txt += 'echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
81 slacapra 1.7 txt += 'echo "SyncCE='+self.name()+'.`hostname -d`" | tee -a $RUNTIME_AREA/$repo \n'
82 spiga 1.27
83 fanzago 1.23 txt += 'middleware='+self.name().upper()+' \n'
84 slacapra 1.1
85     txt += 'dumpStatus $RUNTIME_AREA/$repo \n'
86    
87 slacapra 1.12 txt += 'InputSandBox=${args[3]}\n'
88    
89 slacapra 1.1 txt += '\n\n'
90    
91     return txt
92    
93 spiga 1.27 def wsCopyOutput_comm(self, pool=None):
94 slacapra 1.3 """
95     Write a CopyResults part of a job script, e.g.
96     to copy produced output into a storage element.
97     """
98 spiga 1.37 index = int(common._db.nJobs())
99     job = common.job_list[index-1]
100     jbt = job.type()
101 slacapra 1.8 txt = '\n'
102 spiga 1.27 if int(self.copy_data) == 1:
103 slacapra 1.3
104 spiga 1.27 stageout = PhEDExDatasvcInfo(self.cfg_params)
105     endpoint, lfn, SE, SE_PATH, user = stageout.getEndpoint()
106 ewv 1.50 if self.check_RemoteDir == 1 :
107 spiga 1.37 self.checkRemoteDir(endpoint,jbt.outList('list') )
108 slacapra 1.3
109 spiga 1.27 txt += '#\n'
110     txt += '# COPY OUTPUT FILE TO '+SE_PATH+ '\n'
111     txt += '#\n\n'
112    
113 fanzago 1.35 txt += 'export SE='+SE+'\n'
114 spiga 1.27 txt += 'echo "SE = $SE"\n'
115     txt += 'export SE_PATH='+SE_PATH+'\n'
116     txt += 'echo "SE_PATH = $SE_PATH"\n'
117     txt += 'export LFNBaseName='+lfn+'\n'
118 spiga 1.33 txt += 'echo "LFNBaseName = $LFNBaseName"\n'
119 spiga 1.27 txt += 'export USER='+user+'\n'
120 spiga 1.33 txt += 'echo "USER = $USER"\n'
121 spiga 1.27 txt += 'export endpoint='+endpoint+'\n'
122 spiga 1.33 txt += 'echo "endpoint = $endpoint"\n'
123 fanzago 1.36 ### Needed i.e. for caf
124     if (pool) and (pool != 'None'):
125 spiga 1.27 txt += 'export STAGE_SVCCLASS='+str(pool)+'\n'
126 ewv 1.50
127 spiga 1.27 txt += 'echo ">>> Copy output files from WN = `hostname` to $SE_PATH :"\n'
128     txt += 'export TIME_STAGEOUT_INI=`date +%s` \n'
129     txt += 'copy_exit_status=0\n'
130 spiga 1.48 cmscp_args = ' --destination $endpoint --inputFileList $file_list'
131     cmscp_args +=' --middleware $middleware --lfn $LFNBaseName %s %s '%(self.loc_stage_out,self.debugWrap)
132 spiga 1.46 txt += 'echo "python cmscp.py %s "\n'%cmscp_args
133     txt += 'python cmscp.py %s \n'%cmscp_args
134 ewv 1.50 if self.debug_wrapper==1:
135 fanzago 1.43 txt += 'if [ -f .SEinteraction.log ] ;then\n'
136 spiga 1.27 txt += '########### details of SE interaction\n'
137 spiga 1.39 txt += ' cat .SEinteraction.log\n'
138     txt += 'else\n'
139     txt += ' echo ".SEinteraction.log file not found"\n'
140 spiga 1.40 txt += 'fi\n'
141 spiga 1.45 txt += '##################################### \n'
142 spiga 1.40 txt += 'if [ -f cmscpReport.sh ] ;then\n'
143 fanzago 1.43 txt += ' cat cmscpReport.sh\n'
144 spiga 1.40 txt += ' source cmscpReport.sh\n'
145     txt += 'else\n'
146 ewv 1.50 txt += ' echo "cmscpReport.sh file not found"\n'
147 spiga 1.40 txt += ' StageOutExitStatus=60307\n'
148     txt += 'fi\n'
149 spiga 1.27 txt += 'if [ $StageOutExitStatus -ne 0 ]; then\n'
150     txt += ' echo "Problem copying file to $SE $SE_PATH"\n'
151     txt += ' copy_exit_status=$StageOutExitStatus \n'
152 ewv 1.50 if not self.debug_wrapper==1:
153 spiga 1.32 txt += ' ########### details of SE interaction\n'
154 spiga 1.40 txt += ' if [ -f .SEinteraction.log ] ;then\n'
155     txt += ' cat .SEinteraction.log\n'
156     txt += ' else\n'
157     txt += ' echo ".SEinteraction.log file not found"\n'
158     txt += ' fi\n'
159 spiga 1.45 txt += ' ##################################### \n'
160 spiga 1.27 txt += ' job_exit_code=$StageOutExitStatus\n'
161     txt += 'fi\n'
162     txt += 'export TIME_STAGEOUT_END=`date +%s` \n'
163     txt += 'let "TIME_STAGEOUT = TIME_STAGEOUT_END - TIME_STAGEOUT_INI" \n'
164     else:
165     txt += 'export TIME_STAGEOUT=-1 \n'
166 slacapra 1.3 return txt
167 spiga 1.21
168     def wsExitFunc_comm(self):
169     """
170     """
171     txt = ''
172     txt += ' if [ $PYTHONPATH ]; then \n'
173     txt += ' if [ ! -s $RUNTIME_AREA/fillCrabFjr.py ]; then \n'
174     txt += ' echo "WARNING: it is not possible to create crab_fjr.xml to final report" \n'
175     txt += ' else \n'
176     txt += ' python $RUNTIME_AREA/fillCrabFjr.py $RUNTIME_AREA/crab_fjr_$NJob.xml --errorcode $job_exit_code $executable_exit_status \n'
177     txt += ' fi\n'
178     txt += ' fi\n'
179 ewv 1.50 txt += ' cd $RUNTIME_AREA \n'
180 spiga 1.21 txt += ' for file in $filesToCheck ; do\n'
181     txt += ' if [ -e $file ]; then\n'
182     txt += ' echo "tarring file $file in $out_files"\n'
183     txt += ' else\n'
184     txt += ' echo "WARNING: output file $file not found!"\n'
185     txt += ' fi\n'
186     txt += ' done\n'
187     txt += ' export TIME_WRAP_END=`date +%s`\n'
188     txt += ' let "TIME_WRAP = TIME_WRAP_END - TIME_WRAP_END_INI" \n'
189     txt += ' if [ $PYTHONPATH ]; then \n'
190     txt += ' if [ ! -s $RUNTIME_AREA/fillCrabFjr.py ]; then \n'
191     txt += ' echo "WARNING: it is not possible to create crab_fjr.xml to final report" \n'
192     txt += ' else \n'
193     # call timing FJR filling
194     txt += ' python $RUNTIME_AREA/fillCrabFjr.py $RUNTIME_AREA/crab_fjr_$NJob.xml --timing $TIME_WRAP $TIME_EXE $TIME_STAGEOUT \n'
195 ewv 1.50 txt += ' echo "CrabWrapperTime=$TIME_WRAP" >> $RUNTIME_AREA/$repo \n'
196 spiga 1.21 txt += ' if [ $TIME_STAGEOUT -lt 0 ]; then \n'
197     txt += ' export TIME_STAGEOUT=NULL \n'
198     txt += ' fi\n'
199     txt += ' echo "CrabStageoutTime=$TIME_STAGEOUT" >> $RUNTIME_AREA/$repo \n'
200     txt += ' fi\n'
201     txt += ' fi\n'
202     txt += ' dumpStatus $RUNTIME_AREA/$repo \n\n'
203     txt += ' echo "JOB_EXIT_STATUS = $job_exit_code"\n'
204     txt += ' echo "JobExitCode=$job_exit_code" >> $RUNTIME_AREA/$repo\n'
205     txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
206    
207     return txt
208