ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/SchedulerCondor.py

Comparing COMP/CRAB/python/SchedulerCondor.py (file contents):
Revision 1.4 by ewv, Wed May 28 19:53:29 2008 UTC vs.
Revision 1.16 by ewv, Mon Jan 26 19:43:48 2009 UTC

#	Line 1 | Line 1
1	+	"""
2	+	Implements the vanilla (local) Condor scheduler
3	+	"""
4	+
5		__revision__ = "$Id$"
6		__version__ = "$Revision$"
7
8	<	from Scheduler import Scheduler
9	<	from SchedulerLocal import SchedulerLocal
6	<	from crab_exceptions import *
7	<	from crab_util import *
8	<	from crab_logger import Logger
9	<	import common
8	>	from SchedulerLocal  import SchedulerLocal
9	>	from crab_exceptions import CrabException
10
11	+	import common
12		import os
13
13	–	#  Naming convention:
14	–	#  methods starting with 'ws' are responsible to provide
15	–	#  corresponding part of the job script ('ws' stands for 'write script').
14
15		class SchedulerCondor(SchedulerLocal) :
18	–
19	–	def __init__(self):
20	–	Scheduler.__init__(self,"CONDOR")
21	–	return
22	–
23	–
24	–	def configure(self, cfg_params):
25	–	SchedulerLocal.configure(self, cfg_params)
26	–	self.environment_unique_identifier ='${HOSTNAME}_${CONDOR_ID}_' + common._db.queryTask('name')
27	–	return
28	–
29	–
30	–	def sched_parameter(self,i,task):
31	–	"""
32	–	Return scheduler-specific parameters
16		"""
17	<	index = int(common._db.nJobs()) - 1
18	<	sched_param= ''
19	<
20	<	for i in range(index):
21	<	pass
22	<
23	<	return sched_param
24	<
25	<
26	<	def realSchedParams(self,cfg_params):
27	<	"""
28	<	Return dictionary with specific parameters, to use
29	<	with real scheduler
30	<	"""
31	<
32	<	tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
33	<	params = {'tmpDir':tmpDir}
34	<	return  params
35	<
36	<
37	<	def decodeLogInfo(self, file):
38	<	"""
39	<	Parse logging info file and return main info
40	<	"""
41	<	import CondorGLoggingInfo
42	<	loggingInfo = CondorGLoggingInfo.CondorGLoggingInfo()
43	<	reason = loggingInfo.decodeReason(file)
44	<	return reason
45	<
46	<
47	<	def wsExitFunc(self):
48	<	"""
49	<	"""
50	<	txt = '\n'
51	<	txt += '#\n'
52	<	txt += '# EXECUTE THIS FUNCTION BEFORE EXIT \n'
53	<	txt += '#\n\n'
54	<
55	<	txt += 'func_exit() { \n'
56	<	txt += '    if [ $PYTHONPATH ]; then \n'
57	<	txt += '        update_fjr\n'
58	<	txt += '    fi\n'
59	<	txt += '    for file in $filesToCheck ; do\n'
60	<	txt += '        if [ -e $file ]; then\n'
61	<	txt += '            echo "tarring file $file in  $out_files"\n'
62	<	txt += '        else\n'
63	<	txt += '            echo "WARNING: output file $file not found!"\n'
64	<	txt += '        fi\n'
65	<	txt += '    done\n'
66	<	txt += '    final_list=$filesToCheck\n'
67	<	txt += '    echo "JOB_EXIT_STATUS = $job_exit_code"\n'
68	<	txt += '    echo "JobExitCode=$job_exit_code" >> $RUNTIME_AREA/$repo\n'
69	<	txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
70	<	txt += '    tar zcvf ${out_files}.tgz  ${final_list}\n'
71	<	txt += '    cp  ${out_files}.tgz $ORIG_WD/\n'
72	<	txt += '    cp  crab_fjr_$NJob.xml $ORIG_WD/\n'
73	<
74	<	txt += '    exit $job_exit_code\n'
75	<	txt += '}\n'
17	>	Class to implement the vanilla (local) Condor scheduler
18	>	Naming convention:  Methods starting with 'ws' provide
19	>	the corresponding part of the job script
20	>	('ws' stands for 'write script').
21	>	"""
22	>
23	>	def __init__(self):
24	>	SchedulerLocal.__init__(self,"CONDOR")
25	>	self.datasetPath   = None
26	>	self.selectNoInput = None
27	>	self.environment_unique_identifier = None
28	>	return
29	>
30	>
31	>	def configure(self, cfg_params):
32	>	"""
33	>	Configure the scheduler with the config settings from the user
34	>	"""
35	>
36	>	SchedulerLocal.configure(self, cfg_params)
37	>	self.environment_unique_identifier = '${HOSTNAME}_${CONDOR_ID}_' \
38	>	+ common._db.queryTask('name')
39	>
40	>	try:
41	>	tmp =  cfg_params['CMSSW.datasetpath']
42	>	if tmp.lower() == 'none':
43	>	self.datasetPath = None
44	>	self.selectNoInput = 1
45	>	else:
46	>	self.datasetPath = tmp
47	>	self.selectNoInput = 0
48	>	except KeyError:
49	>	msg = "Error: datasetpath not defined "
50	>	raise CrabException(msg)
51	>
52	>	self.return_data = cfg_params.get('USER.return_data', 0)
53	>	self.copy_data   = cfg_params.get("USER.copy_data", 0)
54	>
55	>	if ( int(self.return_data) == 0 and int(self.copy_data) == 0 ):
56	>	msg = 'Error: return_data and copy_data cannot be set both to 0\n'
57	>	msg = msg + 'Please modify your crab.cfg file\n'
58	>	raise CrabException(msg)
59	>
60	>	if ( int(self.return_data) == 1 and int(self.copy_data) == 1 ):
61	>	msg = 'Error: return_data and copy_data cannot be set both to 1\n'
62	>	msg = msg + 'Please modify your crab.cfg file\n'
63	>	raise CrabException(msg)
64	>
65	>	if ( int(self.copy_data) == 0 and int(self.publish_data) == 1 ):
66	>	msg = 'Warning: publish_data = 1 must be used with copy_data = 1\n'
67	>	msg = msg + 'Please modify copy_data value in your crab.cfg file\n'
68	>	common.logger.message(msg)
69	>	raise CrabException(msg)
70	>
71	>	if int(self.copy_data) == 1:
72	>	self.SE = cfg_params.get('USER.storage_element', None)
73	>	if not self.SE:
74	>	msg = "Error. The [USER] section has no 'storage_element'"
75	>	common.logger.message(msg)
76	>	raise CrabException(msg)
77	>
78	>	self.proxyValid = 0
79	>	self.dontCheckProxy = int(cfg_params.get("EDG.dont_check_proxy",0))
80	>	self.proxyServer = cfg_params.get("EDG.proxy_server",'myproxy.cern.ch')
81	>	common.logger.debug(5,'Setting myproxy server to ' + self.proxyServer)
82	>
83	>	self.group = cfg_params.get("EDG.group", None)
84	>	self.role  = cfg_params.get("EDG.role", None)
85	>	self.VO    = cfg_params.get('EDG.virtual_organization', 'cms')
86	>
87	>	self.checkProxy()
88	>	self.role  = None
89	>
90	>	return
91	>
92	>
93	>	def sched_parameter(self, i, task):
94	>	"""
95	>	Return scheduler-specific parameters
96	>	"""
97	>
98	>	index = int(common._db.nJobs()) - 1
99	>	schedParam = ''
100	>
101	>	for i in range(index):
102	>	pass
103	>
104	>	return schedParam
105	>
106	>
107	>	def realSchedParams(self, cfg_params):
108	>	"""
109	>	Return dictionary with specific parameters, to use with real scheduler
110	>	"""
111	>
112	>	tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
113	>	tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
114	>	jobDir = common.work_space.jobDir()
115	>	params = {'tmpDir':tmpDir,
116	>	'jobDir':jobDir}
117	>	return params
118	>
119	>
120	>	def listMatch(self, seList, full):
121	>	"""
122	>	Check the compatibility of available resources
123	>	"""
124	>
125	>	return [True]
126	>
127	>
128	>	def decodeLogInfo(self, fileName):
129	>	"""
130	>	Parse logging info file and return main info
131	>	"""
132	>
133	>	import CondorGLoggingInfo
134	>	loggingInfo = CondorGLoggingInfo.CondorGLoggingInfo()
135	>	reason = loggingInfo.decodeReason(fileName)
136	>	return reason
137	>
138	>
139	>	def wsCopyOutput(self):
140	>	"""
141	>	Write a CopyResults part of a job script, e.g.
142	>	to copy produced output into a storage element.
143	>	"""
144	>	txt = self.wsCopyOutput_comm()
145	>	return txt
146	>
147	>
148	>	def wsExitFunc(self):
149	>	"""
150	>	Returns the part of the job script which runs prior to exit
151	>	"""
152	>
153	>	txt = '\n'
154	>	txt += '#\n'
155	>	txt += '# EXECUTE THIS FUNCTION BEFORE EXIT \n'
156	>	txt += '#\n\n'
157	>
158	>	txt += 'func_exit() { \n'
159	>	txt += self.wsExitFunc_common()
160	>
161	>	txt += '    tar zcvf ${out_files}.tgz  ${final_list}\n'
162	>	txt += '    cp  ${out_files}.tgz $_CONDOR_SCRATCH_DIR/\n'
163	>	txt += '    cp  crab_fjr_$NJob.xml $_CONDOR_SCRATCH_DIR/\n'
164	>
165	>	txt += '    exit $job_exit_code\n'
166	>	txt += '}\n'
167	>
168	>	return txt
169	>
170	>	def wsInitialEnvironment(self):
171	>	"""
172	>	Returns part of a job script which does scheduler-specific work.
173	>	"""
174	>
175	>	txt  = '\n# Written by SchedulerCondor::wsInitialEnvironment\n'
176	>	txt += 'echo "Beginning environment"\n'
177	>	txt += 'printenv | sort\n'
178	>
179	>	txt += 'middleware='+self.name()+' \n'
180	>	txt += 'if [ -e /opt/d-cache/srm/bin ]; then\n'
181	>	txt += '  export PATH=${PATH}:/opt/d-cache/srm/bin\n'
182	>	txt += 'fi\n'
183	>
184	>	txt += """
185	>	if [ $_CONDOR_SCRATCH_DIR ] && [ -d $_CONDOR_SCRATCH_DIR ]; then
186	>	echo "cd to Condor scratch directory: $_CONDOR_SCRATCH_DIR"
187	>	if [ -e ../default.tgz ] ;then
188	>	echo "Found ISB in parent directory (Local Condor)"
189	>	cp ../default.tgz $_CONDOR_SCRATCH_DIR
190	>	fi
191	>	cd $_CONDOR_SCRATCH_DIR
192	>	fi
193	>	"""
194
195	<	return txt
195	>	return txt

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines