ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/SchedulerLocal.py
Revision: 1.22
Committed: Fri Jun 20 08:57:37 2008 UTC (16 years, 10 months ago) by farinafa
Content type: text/x-python
Branch: MAIN
Changes since 1.21: +1 -1 lines
Log Message: 
Stronger assumption on the stageout timing initialization

File Contents

# User Rev Content
1 slacapra 1.1 from Scheduler import Scheduler
2     from crab_exceptions import *
3     from crab_logger import Logger
4     import common
5    
6     import os,string
7    
8 slacapra 1.2 # Base class for all local scheduler
9 slacapra 1.1
10     class SchedulerLocal(Scheduler) :
11    
12     def configure(self, cfg_params):
13    
14     self.jobtypeName = cfg_params['CRAB.jobtype']
15    
16     name=string.upper(self.name())
17     self.queue = cfg_params.get(name+'.queue',None)
18    
19     self.res = cfg_params.get(name+'.resource',None)
20    
21     if (cfg_params.has_key(self.name()+'.env_id')): self.environment_unique_identifier = cfg_params[self.name()+'.env_id']
22    
23 spiga 1.14 self._taskId=str("_".join(common._db.queryTask('name').split('_')[:-1]))
24 slacapra 1.1
25 slacapra 1.3 self.return_data = int(cfg_params.get('USER.return_data',0))
26    
27     self.copy_data = int(cfg_params.get("USER.copy_data",0))
28     if self.copy_data == 1:
29 ewv 1.20 self._copyCommand = cfg_params.get('USER.copycommand','rfcp')
30     common.logger.debug(3, "copyCommand set to "+ self._copyCommand)
31 slacapra 1.3 self.SE_path= cfg_params.get('USER.storage_path',None)
32     if not self.SE_path:
33     if os.environ.has_key('CASTOR_HOME'):
34     self.SE_path=os.environ['CASTOR_HOME']
35     else:
36     msg='No USER.storage_path has been provided: cannot copy_output'
37     raise CrabException(msg)
38     pass
39     pass
40 slacapra 1.10 self.SE_path+='/'
41 slacapra 1.3
42     if ( self.return_data == 0 and self.copy_data == 0 ):
43     msg = 'Error: return_data = 0 and copy_data = 0 ==> your exe output will be lost\n'
44     msg = msg + 'Please modify return_data and copy_data value in your crab.cfg file\n'
45     raise CrabException(msg)
46    
47     if ( self.return_data == 1 and self.copy_data == 1 ):
48     msg = 'Error: return_data and copy_data cannot be set both to 1\n'
49     msg = msg + 'Please modify return_data or copy_data value in your crab.cfg file\n'
50     raise CrabException(msg)
51 slacapra 1.1
52     ## Get local domain name
53     import socket
54     tmp=socket.gethostname()
55     dot=string.find(tmp,'.')
56     if (dot==-1):
57     msg='Unkown domain name. Cannot use local scheduler'
58     raise CrabException(msg)
59     localDomainName = string.split(tmp,'.',1)[-1]
60 slacapra 1.11 #common.taskDB.setDict('localSite',localDomainName)
61 slacapra 1.1 ## is this ok?
62 slacapra 1.16 if not cfg_params.has_key('EDG.se_white_list'):
63     cfg_params['EDG.se_white_list']=localDomainName
64     common.logger.message("Your domain name is "+str(localDomainName)+": only local dataset will be considered")
65     else:
66     common.logger.message("Your se_white_list is set to "+str(cfg_params['EDG.se_white_list'])+": only local dataset will be considered")
67    
68 ewv 1.20
69    
70 slacapra 1.1
71 slacapra 1.13 Scheduler.configure(self,cfg_params)
72 slacapra 1.1 return
73    
74     def userName(self):
75     """ return the user name """
76 slacapra 1.5 import pwd,getpass
77     tmp=pwd.getpwnam(getpass.getuser())[4]
78     return "/CN="+tmp.strip()
79 slacapra 1.1
80     def wsSetupEnvironment(self):
81     """
82     Returns part of a job script which does scheduler-specific work.
83     """
84     if not self.environment_unique_identifier:
85     raise CrabException('environment_unique_identifier not set')
86    
87 slacapra 1.12 index = int(common._db.nJobs())
88     job = common.job_list[index-1]
89     jbt = job.type()
90 ewv 1.20 # start with wrapper timing
91 spiga 1.18 txt = 'export TIME_WRAP_INI=`date +%s` \n'
92 farinafa 1.22 txt += 'export TIME_STAGEOUT=-2 \n\n'
93 spiga 1.17
94 spiga 1.19 txt += '# '+self.name()+' specific stuff\n'
95 slacapra 1.1 txt += '# strip arguments\n'
96     txt += 'echo "strip arguments"\n'
97     txt += 'args=("$@")\n'
98     txt += 'nargs=$#\n'
99     txt += 'shift $nargs\n'
100     txt += "# job number (first parameter for job wrapper)\n"
101 ewv 1.9 txt += "NJob=${args[0]}; export NJob\n"
102 slacapra 1.1
103 slacapra 1.12 txt += "out_files=out_files_${NJob}; export out_files\n"
104     txt += "echo $out_files\n"
105     txt += jbt.outList()
106    
107 slacapra 1.6 txt += 'SyncGridJobId=`echo '+self.environment_unique_identifier+'`\n'
108     txt += 'MonitorJobID=`echo ${NJob}_${SyncGridJobId}`\n'
109 slacapra 1.1 txt += 'MonitorID=`echo ' + self._taskId + '`\n'
110    
111     txt += 'echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
112     txt += 'echo "SyncGridJobId=`echo $SyncGridJobId`" | tee -a $RUNTIME_AREA/$repo \n'
113     txt += 'echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
114 slacapra 1.7 txt += 'echo "SyncCE='+self.name()+'.`hostname -d`" | tee -a $RUNTIME_AREA/$repo \n'
115 slacapra 1.1
116     txt += 'middleware='+self.name()+' \n'
117    
118     txt += 'dumpStatus $RUNTIME_AREA/$repo \n'
119    
120 slacapra 1.12 txt += 'InputSandBox=${args[3]}\n'
121    
122 slacapra 1.1 txt += '\n\n'
123    
124     return txt
125    
126 slacapra 1.3 def wsCopyOutput(self):
127     """
128     Write a CopyResults part of a job script, e.g.
129     to copy produced output into a storage element.
130     """
131 slacapra 1.8 txt = '\n'
132     if not self.copy_data: return txt
133 slacapra 1.3
134    
135     txt += '#\n'
136     txt += '# COPY OUTPUT FILE TO '+self.SE_path
137     txt += '#\n\n'
138    
139     txt += 'export SE_PATH='+self.SE_path+'\n'
140    
141     txt += 'export CP_CMD='+self._copyCommand+'\n'
142 ewv 1.20 common.logger.debug(3, "Wrapper script CP_CMD set to "+ self._copyCommand)
143 slacapra 1.3
144     txt += 'echo ">>> Copy output files from WN = `hostname` to PATH = $SE_PATH using $CP_CMD :"\n'
145    
146 spiga 1.15 txt += 'if [ $job_exit_code -eq 60302 ]; then\n'
147 slacapra 1.3 txt += ' echo "--> No output file to copy to $SE"\n'
148 spiga 1.15 txt += ' copy_exit_status=$job_exit_code\n'
149 slacapra 1.3 txt += ' echo "COPY_EXIT_STATUS = $copy_exit_status"\n'
150     txt += 'else\n'
151     txt += ' for out_file in $file_list ; do\n'
152     txt += ' echo "Trying to copy output file to $SE_PATH"\n'
153     txt += ' $CP_CMD $SOFTWARE_DIR/$out_file ${SE_PATH}/$out_file\n'
154     txt += ' copy_exit_status=$?\n'
155     txt += ' echo "COPY_EXIT_STATUS = $copy_exit_status"\n'
156     txt += ' echo "STAGE_OUT = $copy_exit_status"\n'
157     txt += ' if [ $copy_exit_status -ne 0 ]; then\n'
158     txt += ' echo "Problem copying $out_file to $SE $SE_PATH"\n'
159     txt += ' echo "StageOutExitStatus = $copy_exit_status " | tee -a $RUNTIME_AREA/$repo\n'
160 fanzago 1.4 #txt += ' copy_exit_status=60307\n'
161 slacapra 1.3 txt += ' else\n'
162     txt += ' echo "StageOutSE = $SE" | tee -a $RUNTIME_AREA/$repo\n'
163     txt += ' echo "StageOutCatalog = " | tee -a $RUNTIME_AREA/$repo\n'
164     txt += ' echo "output copied into $SE/$SE_PATH directory"\n'
165     txt += ' echo "StageOutExitStatus = 0" | tee -a $RUNTIME_AREA/$repo\n'
166     txt += ' fi\n'
167     txt += ' done\n'
168     txt += 'fi\n'
169     txt += 'exit_status=$copy_exit_status\n'
170    
171     return txt
172 spiga 1.21
173    
174     def wsExitFunc_comm(self):
175     """
176     """
177     txt = ''
178     txt += ' if [ $PYTHONPATH ]; then \n'
179     txt += ' if [ ! -s $RUNTIME_AREA/fillCrabFjr.py ]; then \n'
180     txt += ' echo "WARNING: it is not possible to create crab_fjr.xml to final report" \n'
181     txt += ' else \n'
182     txt += ' python $RUNTIME_AREA/fillCrabFjr.py $RUNTIME_AREA/crab_fjr_$NJob.xml --errorcode $job_exit_code $executable_exit_status \n'
183     txt += ' fi\n'
184     txt += ' fi\n'
185     txt += ' cd $RUNTIME_AREA \n'
186     txt += ' for file in $filesToCheck ; do\n'
187     txt += ' if [ -e $file ]; then\n'
188     txt += ' echo "tarring file $file in $out_files"\n'
189     txt += ' else\n'
190     txt += ' echo "WARNING: output file $file not found!"\n'
191     txt += ' fi\n'
192     txt += ' done\n'
193     txt += ' export TIME_WRAP_END=`date +%s`\n'
194     txt += ' let "TIME_WRAP = TIME_WRAP_END - TIME_WRAP_END_INI" \n'
195     txt += ' if [ $PYTHONPATH ]; then \n'
196     txt += ' if [ ! -s $RUNTIME_AREA/fillCrabFjr.py ]; then \n'
197     txt += ' echo "WARNING: it is not possible to create crab_fjr.xml to final report" \n'
198     txt += ' else \n'
199     # call timing FJR filling
200     txt += ' python $RUNTIME_AREA/fillCrabFjr.py $RUNTIME_AREA/crab_fjr_$NJob.xml --timing $TIME_WRAP $TIME_EXE $TIME_STAGEOUT \n'
201     txt += ' echo "CrabWrapperTime=$TIME_WRAP" >> $RUNTIME_AREA/$repo \n'
202     txt += ' if [ $TIME_STAGEOUT -lt 0 ]; then \n'
203     txt += ' export TIME_STAGEOUT=NULL \n'
204     txt += ' fi\n'
205     txt += ' echo "CrabStageoutTime=$TIME_STAGEOUT" >> $RUNTIME_AREA/$repo \n'
206     txt += ' fi\n'
207     txt += ' fi\n'
208     txt += ' dumpStatus $RUNTIME_AREA/$repo \n\n'
209     txt += ' echo "JOB_EXIT_STATUS = $job_exit_code"\n'
210     txt += ' echo "JobExitCode=$job_exit_code" >> $RUNTIME_AREA/$repo\n'
211     txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
212    
213     return txt
214