ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/SchedulerLocal.py
Revision: 1.57
Committed: Tue Dec 15 13:13:40 2009 UTC (15 years, 4 months ago) by farinafa
Content type: text/x-python
Branch: MAIN
CVS Tags: CRAB_2_7_1_pre4, CRAB_2_7_1_pre3, CRAB_2_7_1_pre2, CRAB_2_7_1_pre1
Changes since 1.56: +3 -0 lines
Log Message:
Fix for bug #58043: now stageout uses unique names for the files sent to SE

File Contents

# User Rev Content
1 slacapra 1.1 from Scheduler import Scheduler
2     from crab_exceptions import *
3 spiga 1.47 from crab_util import getLocalDomain
4 slacapra 1.1 import common
5 spiga 1.27 from PhEDExDatasvcInfo import PhEDExDatasvcInfo
6 slacapra 1.1
7     import os,string
8    
9 slacapra 1.2 # Base class for all local scheduler
10 slacapra 1.1
11     class SchedulerLocal(Scheduler) :
12    
13     def configure(self, cfg_params):
14 spiga 1.47 self.environment_unique_identifier = None
15 spiga 1.27 self.cfg_params = cfg_params
16 spiga 1.46 Scheduler.configure(self,cfg_params)
17 slacapra 1.1 self.jobtypeName = cfg_params['CRAB.jobtype']
18    
19     name=string.upper(self.name())
20     self.queue = cfg_params.get(name+'.queue',None)
21    
22     self.res = cfg_params.get(name+'.resource',None)
23    
24 farinafa 1.55 # minimal padding time for jobs. For local schedulers is disabled.
25     # Added for alignment purpose only (and for test) with Grid schedulers
26     self.minimal_job_duration = 0
27 farinafa 1.53
28 slacapra 1.1 if (cfg_params.has_key(self.name()+'.env_id')): self.environment_unique_identifier = cfg_params[self.name()+'.env_id']
29     ## is this ok?
30 spiga 1.25 localDomainName = getLocalDomain(self)
31 spiga 1.52 if not cfg_params.has_key('GRID.se_white_list'):
32     cfg_params['GRID.se_white_list']=localDomainName
33 spiga 1.51 common.logger.info("Your domain name is "+str(localDomainName)+": only local dataset will be considered")
34 slacapra 1.16 else:
35 spiga 1.52 common.logger.info("Your se_white_list is set to "+str(cfg_params['GRID.se_white_list'])+": only local dataset will be considered")
36 slacapra 1.1 return
37    
38     def userName(self):
39     """ return the user name """
40 slacapra 1.5 import pwd,getpass
41     tmp=pwd.getpwnam(getpass.getuser())[4]
42     return "/CN="+tmp.strip()
43 slacapra 1.1
44 ewv 1.50 def envUniqueID(self):
45 spiga 1.47 return
46    
47 slacapra 1.1 def wsSetupEnvironment(self):
48     """
49     Returns part of a job script which does scheduler-specific work.
50     """
51 spiga 1.47 taskId = common._db.queryTask('name')
52 slacapra 1.1 if not self.environment_unique_identifier:
53 spiga 1.47 try :
54 ewv 1.50 self.environment_unique_identifier = self.envUniqueID()
55     except :
56 spiga 1.47 raise CrabException('environment_unique_identifier not set')
57 slacapra 1.12 index = int(common._db.nJobs())
58     job = common.job_list[index-1]
59     jbt = job.type()
60 ewv 1.20 # start with wrapper timing
61 spiga 1.18 txt = 'export TIME_WRAP_INI=`date +%s` \n'
62 farinafa 1.22 txt += 'export TIME_STAGEOUT=-2 \n\n'
63 spiga 1.17
64 spiga 1.19 txt += '# '+self.name()+' specific stuff\n'
65 slacapra 1.1 txt += '# strip arguments\n'
66     txt += 'echo "strip arguments"\n'
67     txt += 'args=("$@")\n'
68     txt += 'nargs=$#\n'
69     txt += 'shift $nargs\n'
70     txt += "# job number (first parameter for job wrapper)\n"
71 ewv 1.9 txt += "NJob=${args[0]}; export NJob\n"
72 farinafa 1.57 txt += "NResub=${args[1]}; export NResub\n"
73     txt += 'OutUniqueID=_$NResub\n'
74     txt += 'OutUniqueID=$NJob$OutUniqueID; export OutUniqueID\n'
75 slacapra 1.1
76 slacapra 1.12 txt += "out_files=out_files_${NJob}; export out_files\n"
77     txt += "echo $out_files\n"
78     txt += jbt.outList()
79    
80 slacapra 1.6 txt += 'SyncGridJobId=`echo '+self.environment_unique_identifier+'`\n'
81     txt += 'MonitorJobID=`echo ${NJob}_${SyncGridJobId}`\n'
82 spiga 1.38 txt += 'MonitorID=`echo ' + taskId + '`\n'
83 slacapra 1.1
84     txt += 'echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
85     txt += 'echo "SyncGridJobId=`echo $SyncGridJobId`" | tee -a $RUNTIME_AREA/$repo \n'
86     txt += 'echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
87 slacapra 1.7 txt += 'echo "SyncCE='+self.name()+'.`hostname -d`" | tee -a $RUNTIME_AREA/$repo \n'
88 spiga 1.27
89 fanzago 1.23 txt += 'middleware='+self.name().upper()+' \n'
90 slacapra 1.1
91     txt += 'dumpStatus $RUNTIME_AREA/$repo \n'
92    
93 slacapra 1.12 txt += 'InputSandBox=${args[3]}\n'
94    
95 slacapra 1.1 txt += '\n\n'
96    
97     return txt
98    
99 spiga 1.27 def wsCopyOutput_comm(self, pool=None):
100 slacapra 1.3 """
101     Write a CopyResults part of a job script, e.g.
102     to copy produced output into a storage element.
103     """
104 spiga 1.37 index = int(common._db.nJobs())
105     job = common.job_list[index-1]
106     jbt = job.type()
107 slacapra 1.8 txt = '\n'
108 spiga 1.27 if int(self.copy_data) == 1:
109 slacapra 1.3
110 spiga 1.27 stageout = PhEDExDatasvcInfo(self.cfg_params)
111     endpoint, lfn, SE, SE_PATH, user = stageout.getEndpoint()
112 ewv 1.50 if self.check_RemoteDir == 1 :
113 spiga 1.37 self.checkRemoteDir(endpoint,jbt.outList('list') )
114 slacapra 1.3
115 spiga 1.27 txt += '#\n'
116     txt += '# COPY OUTPUT FILE TO '+SE_PATH+ '\n'
117     txt += '#\n\n'
118    
119 fanzago 1.35 txt += 'export SE='+SE+'\n'
120 spiga 1.27 txt += 'echo "SE = $SE"\n'
121     txt += 'export SE_PATH='+SE_PATH+'\n'
122     txt += 'echo "SE_PATH = $SE_PATH"\n'
123     txt += 'export LFNBaseName='+lfn+'\n'
124 spiga 1.33 txt += 'echo "LFNBaseName = $LFNBaseName"\n'
125 spiga 1.27 txt += 'export USER='+user+'\n'
126 spiga 1.33 txt += 'echo "USER = $USER"\n'
127 spiga 1.27 txt += 'export endpoint='+endpoint+'\n'
128 spiga 1.33 txt += 'echo "endpoint = $endpoint"\n'
129 fanzago 1.36 ### Needed i.e. for caf
130     if (pool) and (pool != 'None'):
131 spiga 1.27 txt += 'export STAGE_SVCCLASS='+str(pool)+'\n'
132 ewv 1.50
133 spiga 1.27 txt += 'echo ">>> Copy output files from WN = `hostname` to $SE_PATH :"\n'
134     txt += 'export TIME_STAGEOUT_INI=`date +%s` \n'
135     txt += 'copy_exit_status=0\n'
136 spiga 1.48 cmscp_args = ' --destination $endpoint --inputFileList $file_list'
137     cmscp_args +=' --middleware $middleware --lfn $LFNBaseName %s %s '%(self.loc_stage_out,self.debugWrap)
138 spiga 1.46 txt += 'echo "python cmscp.py %s "\n'%cmscp_args
139     txt += 'python cmscp.py %s \n'%cmscp_args
140 ewv 1.50 if self.debug_wrapper==1:
141 fanzago 1.43 txt += 'if [ -f .SEinteraction.log ] ;then\n'
142 spiga 1.27 txt += '########### details of SE interaction\n'
143 spiga 1.39 txt += ' cat .SEinteraction.log\n'
144     txt += 'else\n'
145     txt += ' echo ".SEinteraction.log file not found"\n'
146 spiga 1.40 txt += 'fi\n'
147 spiga 1.45 txt += '##################################### \n'
148 spiga 1.40 txt += 'if [ -f cmscpReport.sh ] ;then\n'
149 fanzago 1.43 txt += ' cat cmscpReport.sh\n'
150 spiga 1.40 txt += ' source cmscpReport.sh\n'
151     txt += 'else\n'
152 ewv 1.50 txt += ' echo "cmscpReport.sh file not found"\n'
153 spiga 1.40 txt += ' StageOutExitStatus=60307\n'
154     txt += 'fi\n'
155 spiga 1.27 txt += 'if [ $StageOutExitStatus -ne 0 ]; then\n'
156     txt += ' echo "Problem copying file to $SE $SE_PATH"\n'
157     txt += ' copy_exit_status=$StageOutExitStatus \n'
158 ewv 1.50 if not self.debug_wrapper==1:
159 spiga 1.32 txt += ' ########### details of SE interaction\n'
160 spiga 1.40 txt += ' if [ -f .SEinteraction.log ] ;then\n'
161     txt += ' cat .SEinteraction.log\n'
162     txt += ' else\n'
163     txt += ' echo ".SEinteraction.log file not found"\n'
164     txt += ' fi\n'
165 spiga 1.45 txt += ' ##################################### \n'
166 spiga 1.27 txt += ' job_exit_code=$StageOutExitStatus\n'
167     txt += 'fi\n'
168     txt += 'export TIME_STAGEOUT_END=`date +%s` \n'
169     txt += 'let "TIME_STAGEOUT = TIME_STAGEOUT_END - TIME_STAGEOUT_INI" \n'
170     else:
171     txt += 'export TIME_STAGEOUT=-1 \n'
172 slacapra 1.3 return txt
173 spiga 1.21
174 spiga 1.56 def wsExitFunc_common(self):
175 spiga 1.21 """
176     """
177     txt = ''
178     txt += ' if [ $PYTHONPATH ]; then \n'
179     txt += ' if [ ! -s $RUNTIME_AREA/fillCrabFjr.py ]; then \n'
180     txt += ' echo "WARNING: it is not possible to create crab_fjr.xml to final report" \n'
181     txt += ' else \n'
182     txt += ' python $RUNTIME_AREA/fillCrabFjr.py $RUNTIME_AREA/crab_fjr_$NJob.xml --errorcode $job_exit_code $executable_exit_status \n'
183     txt += ' fi\n'
184     txt += ' fi\n'
185 ewv 1.50 txt += ' cd $RUNTIME_AREA \n'
186 spiga 1.21 txt += ' for file in $filesToCheck ; do\n'
187     txt += ' if [ -e $file ]; then\n'
188     txt += ' echo "tarring file $file in $out_files"\n'
189     txt += ' else\n'
190     txt += ' echo "WARNING: output file $file not found!"\n'
191     txt += ' fi\n'
192     txt += ' done\n'
193     txt += ' export TIME_WRAP_END=`date +%s`\n'
194 farinafa 1.53 txt += ' let "TIME_WRAP = TIME_WRAP_END - TIME_WRAP_END_INI" \n\n'
195     # padding for minimal job duration
196     txt += ' let "MIN_JOB_DURATION = 60*%d" \n'%self.minimal_job_duration
197     txt += ' let "PADDING_DURATION = MIN_JOB_DURATION - TIME_WRAP" \n'
198     txt += ' if [ $PADDING_DURATION -gt 0 ]; then \n'
199     txt += ' echo ">>> padding time: Sleeping the wrapper for $PADDING_DURATION seconds"\n'
200     txt += ' sleep $PADDING_DURATION\n'
201     txt += ' TIME_WRAP_END=`date +%s`\n'
202     txt += ' let "TIME_WRAP = TIME_WRAP_END - TIME_WRAP_INI" \n'
203     txt += ' else \n'
204     txt += ' echo ">>> padding time: Wrapper lasting more than $MIN_JOB_DURATION seconds. No sleep required."\n'
205     txt += ' fi\n\n'
206     # call timing FJR filling
207 spiga 1.21 txt += ' if [ $PYTHONPATH ]; then \n'
208     txt += ' if [ ! -s $RUNTIME_AREA/fillCrabFjr.py ]; then \n'
209     txt += ' echo "WARNING: it is not possible to create crab_fjr.xml to final report" \n'
210     txt += ' else \n'
211     txt += ' python $RUNTIME_AREA/fillCrabFjr.py $RUNTIME_AREA/crab_fjr_$NJob.xml --timing $TIME_WRAP $TIME_EXE $TIME_STAGEOUT \n'
212 ewv 1.50 txt += ' echo "CrabWrapperTime=$TIME_WRAP" >> $RUNTIME_AREA/$repo \n'
213 spiga 1.21 txt += ' if [ $TIME_STAGEOUT -lt 0 ]; then \n'
214     txt += ' export TIME_STAGEOUT=NULL \n'
215     txt += ' fi\n'
216     txt += ' echo "CrabStageoutTime=$TIME_STAGEOUT" >> $RUNTIME_AREA/$repo \n'
217     txt += ' fi\n'
218     txt += ' fi\n'
219     txt += ' dumpStatus $RUNTIME_AREA/$repo \n\n'
220     txt += ' echo "JOB_EXIT_STATUS = $job_exit_code"\n'
221     txt += ' echo "JobExitCode=$job_exit_code" >> $RUNTIME_AREA/$repo\n'
222     txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
223    
224     return txt
225