ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/SchedulerCondor.py

Comparing COMP/CRAB/python/SchedulerCondor.py (file contents):
Revision 1.7 by ewv, Tue Jun 10 17:46:00 2008 UTC vs.
Revision 1.30 by ewv, Wed Dec 16 17:40:08 2009 UTC

#	Line 1 | Line 1
1	+	"""
2	+	Implements the vanilla (local) Condor scheduler
3	+	"""
4	+
5		__revision__ = "$Id$"
6		__version__ = "$Revision$"
7
8	<	from Scheduler import Scheduler
9	<	from SchedulerLocal import SchedulerLocal
6	<	from crab_exceptions import *
7	<	from crab_util import *
8	<	from crab_logger import Logger
9	<	import common
8	>	from SchedulerLocal  import SchedulerLocal
9	>	from crab_exceptions import CrabException
10
11	+	import common
12		import os
13	+	import socket
14
15	<	#  Naming convention:
16	<	#  methods starting with 'ws' are responsible to provide
17	<	#  corresponding part of the job script ('ws' stands for 'write script').
15	>	# FUTURE: for python 2.4 & 2.6
16	>	try:
17	>	from hashlib import sha1
18	>	except:
19	>	from sha import sha as sha1
20
21		class SchedulerCondor(SchedulerLocal) :
18	–
19	–	def __init__(self):
20	–	Scheduler.__init__(self,"CONDOR")
21	–	return
22	–
23	–
24	–	def configure(self, cfg_params):
25	–	SchedulerLocal.configure(self, cfg_params)
26	–	self.environment_unique_identifier ='${HOSTNAME}_${CONDOR_ID}_' + common._db.queryTask('name')
27	–
28	–	try:
29	–	tmp =  cfg_params['CMSSW.datasetpath']
30	–	if string.lower(tmp)=='none':
31	–	self.datasetPath = None
32	–	self.selectNoInput = 1
33	–	else:
34	–	self.datasetPath = tmp
35	–	self.selectNoInput = 0
36	–	except KeyError:
37	–	msg = "Error: datasetpath not defined "
38	–	raise CrabException(msg)
39	–
40	–	return
41	–
42	–
43	–	def sched_parameter(self,i,task):
44	–	"""
45	–	Return scheduler-specific parameters
46	–	"""
47	–	index = int(common._db.nJobs()) - 1
48	–	sched_param= ''
49	–
50	–	for i in range(index):
51	–	pass
52	–
53	–	return sched_param
54	–
55	–
56	–	def realSchedParams(self,cfg_params):
57	–	"""
58	–	Return dictionary with specific parameters, to use
59	–	with real scheduler
60	–	"""
61	–
62	–	tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
63	–	params = {'tmpDir':tmpDir}
64	–	return  params
65	–
66	–
67	–	def listMatch(self, seList, full, onlyOSG=True):
22		"""
23	<	Check the compatibility of available resources
24	<	"""
25	<
26	<	# May have problems with onlyOSG being false, probably due to lengths of lists and command line.
27	<	# Either re-write osg_bdii.py with a proper ldap library or break the queries apart
28	<
29	<	#scram = Scram.Scram(None)
30	<	#versionCMSSW = scram.getSWVersion()
31	<	#arch = scram.getArch()
32	<
33	<	if self.selectNoInput:
34	<	return [True]
35	<
36	<
37	<	def decodeLogInfo(self, file):
38	<	"""
39	<	Parse logging info file and return main info
40	<	"""
41	<	import CondorGLoggingInfo
42	<	loggingInfo = CondorGLoggingInfo.CondorGLoggingInfo()
43	<	reason = loggingInfo.decodeReason(file)
44	<	return reason
45	<
46	<
47	<	def wsExitFunc(self):
48	<	"""
49	<	"""
50	<	txt = '\n'
51	<	txt += '#\n'
52	<	txt += '# EXECUTE THIS FUNCTION BEFORE EXIT \n'
53	<	txt += '#\n\n'
54	<
55	<	txt += 'func_exit() { \n'
56	<	txt += '    if [ $PYTHONPATH ]; then \n'
57	<	txt += '        update_fjr\n'
58	<	txt += '    fi\n'
59	<	txt += '    for file in $filesToCheck ; do\n'
60	<	txt += '        if [ -e $file ]; then\n'
61	<	txt += '            echo "tarring file $file in  $out_files"\n'
62	<	txt += '        else\n'
63	<	txt += '            echo "WARNING: output file $file not found!"\n'
64	<	txt += '        fi\n'
65	<	txt += '    done\n'
66	<	txt += '    final_list=$filesToCheck\n'
67	<	txt += '    echo "JOB_EXIT_STATUS = $job_exit_code"\n'
68	<	txt += '    echo "JobExitCode=$job_exit_code" >> $RUNTIME_AREA/$repo\n'
69	<	txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
70	<	txt += '    tar zcvf ${out_files}.tgz  ${final_list}\n'
71	<	txt += '    cp  ${out_files}.tgz $ORIG_WD/\n'
72	<	txt += '    cp  crab_fjr_$NJob.xml $ORIG_WD/\n'
73	<
74	<	txt += '    exit $job_exit_code\n'
75	<	txt += '}\n'
76	<
77	<	return txt
78	<
79	<	def wsInitialEnvironment(self):
80	<	"""
81	<	Returns part of a job script which does scheduler-specific work.
82	<	"""
83	<
84	<	txt  = '\n# Written by SchedulerCondor::wsInitialEnvironment\n'
85	<	txt += 'echo "Beginning environment"\n'
86	<	txt += 'printenv | sort\n'
23	>	Class to implement the vanilla (local) Condor scheduler
24	>	Naming convention:  Methods starting with 'ws' provide
25	>	the corresponding part of the job script
26	>	('ws' stands for 'write script').
27	>	"""
28	>
29	>	def __init__(self):
30	>	SchedulerLocal.__init__(self,"CONDOR")
31	>	self.datasetPath   = None
32	>	self.selectNoInput = None
33	>	self.return_data   = 0
34	>	self.copy_data     = 0
35	>
36	>	self.environment_unique_identifier = None
37	>	return
38	>
39	>
40	>	def configure(self, cfg_params):
41	>	"""
42	>	Configure the scheduler with the config settings from the user
43	>	"""
44	>
45	>	SchedulerLocal.configure(self, cfg_params)
46	>
47	>	try:
48	>	tmp =  cfg_params['CMSSW.datasetpath']
49	>	if tmp.lower() == 'none':
50	>	self.datasetPath = None
51	>	self.selectNoInput = 1
52	>	else:
53	>	self.datasetPath = tmp
54	>	self.selectNoInput = 0
55	>	except KeyError:
56	>	msg = "Error: datasetpath not defined "
57	>	raise CrabException(msg)
58	>
59	>	self.return_data = cfg_params.get('USER.return_data', 0)
60	>	self.copy_data   = cfg_params.get("USER.copy_data", 0)
61	>
62	>	self.proxyValid = 0
63	>	self.dontCheckProxy = int(cfg_params.get("GRID.dont_check_proxy", 0))
64	>	self.proxyServer = cfg_params.get("GRID.proxy_server", 'myproxy.cern.ch')
65	>	common.logger.debug('Setting myproxy server to ' + self.proxyServer)
66	>
67	>	self.group = cfg_params.get("GRID.group", None)
68	>	self.role  = cfg_params.get("GRID.role", None)
69	>	self.VO    = cfg_params.get('GRID.virtual_organization', 'cms')
70	>
71	>	self.checkProxy()
72	>
73	>	return
74	>
75	>	def envUniqueID(self):
76	>	taskHash = sha1(common._db.queryTask('name')).hexdigest()
77	>	id = "https://" + socket.gethostname() + '/' + taskHash + "/${NJob}"
78	>	return id
79	>
80	>	def sched_parameter(self, i, task):
81	>	"""
82	>	Return scheduler-specific parameters
83	>	"""
84	>	req = ''
85	>	if self.EDG_addJdlParam:
86	>	if self.EDG_addJdlParam[-1] == '':
87	>	self.EDG_addJdlParam = self.EDG_addJdlParam[:-1]
88	>	for p in self.EDG_addJdlParam:
89	>	req += p.strip()+';\n'
90	>
91	>	return req
92	>
93	>
94	>	def realSchedParams(self, cfg_params):
95	>	"""
96	>	Return dictionary with specific parameters, to use with real scheduler
97	>	"""
98	>
99	>	tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
100	>	tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
101	>	jobDir = common.work_space.jobDir()
102	>	params = {'tmpDir':tmpDir,
103	>	'jobDir':jobDir}
104	>	return params
105	>
106	>
107	>	def listMatch(self, seList, full):
108	>	"""
109	>	Check the compatibility of available resources
110	>	"""
111	>
112	>	return [True]
113	>
114	>
115	>	def decodeLogInfo(self, fileName):
116	>	"""
117	>	Parse logging info file and return main info
118	>	"""
119	>
120	>	import CondorGLoggingInfo
121	>	loggingInfo = CondorGLoggingInfo.CondorGLoggingInfo()
122	>	reason = loggingInfo.decodeReason(fileName)
123	>	return reason
124	>
125	>
126	>	def wsCopyOutput(self):
127	>	"""
128	>	Write a CopyResults part of a job script, e.g.
129	>	to copy produced output into a storage element.
130	>	"""
131	>	txt = self.wsCopyOutput_comm()
132	>	return txt
133	>
134	>
135	>	def wsExitFunc(self):
136	>	"""
137	>	Returns the part of the job script which runs prior to exit
138	>	"""
139	>
140	>	txt = '\n'
141	>	txt += '#\n'
142	>	txt += '# EXECUTE THIS FUNCTION BEFORE EXIT \n'
143	>	txt += '#\n\n'
144	>
145	>	txt += 'func_exit() { \n'
146	>	txt += self.wsExitFunc_common()
147	>
148	>	txt += '    tar zcvf ${out_files}.tgz  ${final_list}\n'
149	>	txt += '    cp  ${out_files}.tgz $_CONDOR_SCRATCH_DIR/\n'
150	>	txt += '    cp  crab_fjr_$NJob.xml $_CONDOR_SCRATCH_DIR/\n'
151	>
152	>	txt += '    exit $job_exit_code\n'
153	>	txt += '}\n'
154	>
155	>	return txt
156	>
157	>	def wsInitialEnvironment(self):
158	>	"""
159	>	Returns part of a job script which does scheduler-specific work.
160	>	"""
161	>
162	>	txt  = '\n# Written by SchedulerCondor::wsInitialEnvironment\n'
163	>	txt += 'echo "Beginning environment"\n'
164	>	txt += 'printenv | sort\n'
165	>
166	>	txt += 'middleware='+self.name()+' \n'
167	>	txt += 'if [ -e /opt/d-cache/srm/bin ]; then\n'
168	>	txt += '  export PATH=${PATH}:/opt/d-cache/srm/bin\n'
169	>	txt += 'fi\n'
170
171	<	txt += 'middleware='+self.name()+' \n'
135	<	txt += """
171	>	txt += """
172		if [ $_CONDOR_SCRATCH_DIR ] && [ -d $_CONDOR_SCRATCH_DIR ]; then
173	<	ORIG_WD=`pwd`
138	<	echo "Change from $ORIG_WD to Condor scratch directory: $_CONDOR_SCRATCH_DIR"
173	>	echo "cd to Condor scratch directory: $_CONDOR_SCRATCH_DIR"
174		if [ -e ../default.tgz ] ;then
175		echo "Found ISB in parent directory (Local Condor)"
176		cp ../default.tgz $_CONDOR_SCRATCH_DIR
#	Line 144 | Line 179 | if [ $_CONDOR_SCRATCH_DIR ] && [ -d $_CO
179		fi
180		"""
181
182	<	return txt
182	>	return txt
183	>
184	>
185	>	def sched_fix_parameter(self):
186	>	"""
187	>	Returns string with requirements and scheduler-specific parameters
188	>	"""
189	>
190	>	if self.EDG_requirements:
191	>	req = self.EDG_requirements
192	>	taskReq = {'commonRequirements':req}
193	>	common._db.updateTask_(taskReq)

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines