ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/SchedulerCondor.py

Comparing COMP/CRAB/python/SchedulerCondor.py (file contents):
Revision 1.6 by ewv, Thu May 29 19:20:40 2008 UTC vs.
Revision 1.20 by spiga, Fri Mar 6 16:49:55 2009 UTC

#	Line 1 | Line 1
1	+	"""
2	+	Implements the vanilla (local) Condor scheduler
3	+	"""
4	+
5		__revision__ = "$Id$"
6		__version__ = "$Revision$"
7
8	<	from Scheduler import Scheduler
9	<	from SchedulerLocal import SchedulerLocal
6	<	from crab_exceptions import *
7	<	from crab_util import *
8	<	from crab_logger import Logger
9	<	import common
8	>	from SchedulerLocal  import SchedulerLocal
9	>	from crab_exceptions import CrabException
10
11	+	import common
12		import os
13	<
14	<	#  Naming convention:
14	<	#  methods starting with 'ws' are responsible to provide
15	<	#  corresponding part of the job script ('ws' stands for 'write script').
13	>	import socket
14	>	import sha
15
16		class SchedulerCondor(SchedulerLocal) :
17	+	"""
18	+	Class to implement the vanilla (local) Condor scheduler
19	+	Naming convention:  Methods starting with 'ws' provide
20	+	the corresponding part of the job script
21	+	('ws' stands for 'write script').
22	+	"""
23
24	<	def __init__(self):
25	<	Scheduler.__init__(self,"CONDOR")
26	<	return
24	>	def __init__(self):
25	>	SchedulerLocal.__init__(self,"CONDOR")
26	>	self.datasetPath   = None
27	>	self.selectNoInput = None
28	>	self.return_data   = 0
29	>	self.copy_data     = 0
30
31	+	self.environment_unique_identifier = None
32	+	return
33
24	–	def configure(self, cfg_params):
25	–	SchedulerLocal.configure(self, cfg_params)
26	–	self.environment_unique_identifier ='${HOSTNAME}_${CONDOR_ID}_' + common._db.queryTask('name')
27	–	return
34
35	+	def configure(self, cfg_params):
36	+	"""
37	+	Configure the scheduler with the config settings from the user
38	+	"""
39
40	<	def sched_parameter(self,i,task):
41	<	"""
42	<	Return scheduler-specific parameters
43	<	"""
34	<	index = int(common._db.nJobs()) - 1
35	<	sched_param= ''
40	>	SchedulerLocal.configure(self, cfg_params)
41	>	taskHash = sha.new(common._db.queryTask('name')).hexdigest()
42	>	self.environment_unique_identifier = "https://" + \
43	>	socket.gethostname() + '/' + taskHash + "/${NJob}"
44
45	<	for i in range(index):
46	<	pass
45	>	try:
46	>	tmp =  cfg_params['CMSSW.datasetpath']
47	>	if tmp.lower() == 'none':
48	>	self.datasetPath = None
49	>	self.selectNoInput = 1
50	>	else:
51	>	self.datasetPath = tmp
52	>	self.selectNoInput = 0
53	>	except KeyError:
54	>	msg = "Error: datasetpath not defined "
55	>	raise CrabException(msg)
56
57	<	return sched_param
57	>	self.return_data = cfg_params.get('USER.return_data', 0)
58	>	self.copy_data   = cfg_params.get("USER.copy_data", 0)
59
60	+	if int(self.return_data) == 0 and int(self.copy_data) == 0:
61	+	msg =  'Error: return_data and copy_data cannot both be set to 0\n'
62	+	msg += 'Please modify your crab.cfg file\n'
63	+	raise CrabException(msg)
64
65	<	def realSchedParams(self,cfg_params):
66	<	"""
67	<	Return dictionary with specific parameters, to use
68	<	with real scheduler
47	<	"""
65	>	if int(self.return_data) == 1 and int(self.copy_data) == 1:
66	>	msg =  'Error: return_data and copy_data cannot both be set to 1\n'
67	>	msg += 'Please modify your crab.cfg file\n'
68	>	raise CrabException(msg)
69
70	<	tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
71	<	params = {'tmpDir':tmpDir}
72	<	return  params
70	>	if int(self.copy_data) == 0 and int(self.publish_data) == 1:
71	>	msg =  'Warning: publish_data=1 must be used with copy_data=1\n'
72	>	msg += 'Please modify the copy_data value in your crab.cfg file\n'
73	>	common.logger.message(msg)
74	>	raise CrabException(msg)
75
76	+	if int(self.copy_data) == 1:
77	+	self.SE = cfg_params.get('USER.storage_element', None)
78	+	if not self.SE:
79	+	msg = "Error. The [USER] section has no 'storage_element'"
80	+	common.logger.message(msg)
81	+	raise CrabException(msg)
82
83	<	def decodeLogInfo(self, file):
84	<	"""
85	<	Parse logging info file and return main info
86	<	"""
58	<	import CondorGLoggingInfo
59	<	loggingInfo = CondorGLoggingInfo.CondorGLoggingInfo()
60	<	reason = loggingInfo.decodeReason(file)
61	<	return reason
83	>	self.proxyValid = 0
84	>	self.dontCheckProxy = int(cfg_params.get("EDG.dont_check_proxy", 0))
85	>	self.proxyServer = cfg_params.get("EDG.proxy_server", 'myproxy.cern.ch')
86	>	common.logger.debug(5,'Setting myproxy server to ' + self.proxyServer)
87
88	+	self.group = cfg_params.get("EDG.group", None)
89	+	self.role  = cfg_params.get("EDG.role", None)
90	+	self.VO    = cfg_params.get('EDG.virtual_organization', 'cms')
91
92	<	def wsExitFunc(self):
65	<	"""
66	<	"""
67	<	txt = '\n'
68	<	txt += '#\n'
69	<	txt += '# EXECUTE THIS FUNCTION BEFORE EXIT \n'
70	<	txt += '#\n\n'
71	<
72	<	txt += 'func_exit() { \n'
73	<	txt += '    if [ $PYTHONPATH ]; then \n'
74	<	txt += '        update_fjr\n'
75	<	txt += '    fi\n'
76	<	txt += '    for file in $filesToCheck ; do\n'
77	<	txt += '        if [ -e $file ]; then\n'
78	<	txt += '            echo "tarring file $file in  $out_files"\n'
79	<	txt += '        else\n'
80	<	txt += '            echo "WARNING: output file $file not found!"\n'
81	<	txt += '        fi\n'
82	<	txt += '    done\n'
83	<	txt += '    final_list=$filesToCheck\n'
84	<	txt += '    echo "JOB_EXIT_STATUS = $job_exit_code"\n'
85	<	txt += '    echo "JobExitCode=$job_exit_code" >> $RUNTIME_AREA/$repo\n'
86	<	txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
87	<	txt += '    tar zcvf ${out_files}.tgz  ${final_list}\n'
88	<	txt += '    cp  ${out_files}.tgz $ORIG_WD/\n'
89	<	txt += '    cp  crab_fjr_$NJob.xml $ORIG_WD/\n'
92	>	self.checkProxy()
93
94	<	txt += '    exit $job_exit_code\n'
92	<	txt += '}\n'
94	>	self.role  = None
95
96	<	return txt
96	>	return
97
96	–	def wsInitialEnvironment(self):
97	–	"""
98	–	Returns part of a job script which does scheduler-specific work.
99	–	"""
98
99	<	txt  = '\n# Written by SchedulerCondor::wsInitialEnvironment\n'
100	<	txt += 'echo "Beginning environment"\n'
101	<	txt += 'printenv | sort\n'
99	>	def sched_parameter(self, i, task):
100	>	"""
101	>	Return scheduler-specific parameters
102	>	"""
103	>
104	>	index = int(common._db.nJobs()) - 1
105	>	schedParam = ''
106	>
107	>	for i in range(index):
108	>	pass
109	>
110	>	return schedParam
111	>
112	>
113	>	def realSchedParams(self, cfg_params):
114	>	"""
115	>	Return dictionary with specific parameters, to use with real scheduler
116	>	"""
117	>
118	>	tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
119	>	tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
120	>	jobDir = common.work_space.jobDir()
121	>	params = {'tmpDir':tmpDir,
122	>	'jobDir':jobDir}
123	>	return params
124	>
125	>
126	>	def listMatch(self, seList, full):
127	>	"""
128	>	Check the compatibility of available resources
129	>	"""
130	>
131	>	return [True]
132	>
133	>
134	>	def decodeLogInfo(self, fileName):
135	>	"""
136	>	Parse logging info file and return main info
137	>	"""
138	>
139	>	import CondorGLoggingInfo
140	>	loggingInfo = CondorGLoggingInfo.CondorGLoggingInfo()
141	>	reason = loggingInfo.decodeReason(fileName)
142	>	return reason
143	>
144	>
145	>	def wsCopyOutput(self):
146	>	"""
147	>	Write a CopyResults part of a job script, e.g.
148	>	to copy produced output into a storage element.
149	>	"""
150	>	txt = self.wsCopyOutput_comm()
151	>	return txt
152	>
153	>
154	>	def wsExitFunc(self):
155	>	"""
156	>	Returns the part of the job script which runs prior to exit
157	>	"""
158	>
159	>	txt = '\n'
160	>	txt += '#\n'
161	>	txt += '# EXECUTE THIS FUNCTION BEFORE EXIT \n'
162	>	txt += '#\n\n'
163	>
164	>	txt += 'func_exit() { \n'
165	>	txt += self.wsExitFunc_common()
166	>
167	>	txt += '    tar zcvf ${out_files}.tgz  ${final_list}\n'
168	>	txt += '    cp  ${out_files}.tgz $_CONDOR_SCRATCH_DIR/\n'
169	>	txt += '    cp  crab_fjr_$NJob.xml $_CONDOR_SCRATCH_DIR/\n'
170	>
171	>	txt += '    exit $job_exit_code\n'
172	>	txt += '}\n'
173	>
174	>	return txt
175	>
176	>	def wsInitialEnvironment(self):
177	>	"""
178	>	Returns part of a job script which does scheduler-specific work.
179	>	"""
180	>
181	>	txt  = '\n# Written by SchedulerCondor::wsInitialEnvironment\n'
182	>	txt += 'echo "Beginning environment"\n'
183	>	txt += 'printenv | sort\n'
184	>
185	>	txt += 'middleware='+self.name()+' \n'
186	>	txt += 'if [ -e /opt/d-cache/srm/bin ]; then\n'
187	>	txt += '  export PATH=${PATH}:/opt/d-cache/srm/bin\n'
188	>	txt += 'fi\n'
189
190	<	txt += 'middleware='+self.name()+' \n'
106	<	txt += """
190	>	txt += """
191		if [ $_CONDOR_SCRATCH_DIR ] && [ -d $_CONDOR_SCRATCH_DIR ]; then
192	<	ORIG_WD=`pwd`
109	<	echo "Change from $ORIG_WD to Condor scratch directory: $_CONDOR_SCRATCH_DIR"
192	>	echo "cd to Condor scratch directory: $_CONDOR_SCRATCH_DIR"
193		if [ -e ../default.tgz ] ;then
194		echo "Found ISB in parent directory (Local Condor)"
195		cp ../default.tgz $_CONDOR_SCRATCH_DIR
#	Line 115 | Line 198 | if [ $_CONDOR_SCRATCH_DIR ] && [ -d $_CO
198		fi
199		"""
200
201	<	return txt
201	>	return txt

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines