ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/SchedulerCondor.py

Comparing COMP/CRAB/python/SchedulerCondor.py (file contents):
Revision 1.2 by ewv, Wed May 21 19:27:30 2008 UTC vs.
Revision 1.33 by belforte, Thu Jun 14 15:47:28 2012 UTC

#	Line 1 | Line 1
1	+	"""
2	+	Implements the vanilla (local) Condor scheduler
3	+	"""
4	+
5		__revision__ = "$Id$"
6		__version__ = "$Revision$"
7
8	<	from Scheduler import Scheduler
9	<	from SchedulerLocal import SchedulerLocal
6	<	from crab_exceptions import *
7	<	from crab_util import *
8	<	from crab_logger import Logger
9	<	import common
8	>	from SchedulerLocal  import SchedulerLocal
9	>	from crab_exceptions import CrabException
10
11	+	import common
12		import os
13	+	import socket
14
15	<	#  Naming convention:
16	<	#  methods starting with 'ws' are responsible to provide
17	<	#  corresponding part of the job script ('ws' stands for 'write script').
15	>	# FUTURE: for python 2.4 & 2.6
16	>	try:
17	>	from hashlib import sha1
18	>	except:
19	>	from sha import sha as sha1
20
21		class SchedulerCondor(SchedulerLocal) :
18	–
19	–	def __init__(self):
20	–	Scheduler.__init__(self,"CONDOR")
21	–	return
22	–
23	–	def configure(self, cfg_params):
24	–	SchedulerLocal.configure(self, cfg_params)
25	–	self.environment_unique_identifier ='${HOSTNAME}_${CONDOR_ID}_' + common._db.queryTask('name')
26	–	return
27	–
28	–	def sched_parameter(self,i,task):
29	–	"""
30	–	Return scheduler-specific parameters
31	–	"""
32	–	index = int(common._db.nJobs()) - 1
33	–	sched_param= ''
34	–
35	–	for i in range(index):
36	–	pass
37	–
38	–	return sched_param
39	–
40	–	def realSchedParams(self,cfg_params):
22		"""
23	<	Return dictionary with specific parameters, to use
24	<	with real scheduler
23	>	Class to implement the vanilla (local) Condor scheduler
24	>	Naming convention:  Methods starting with 'ws' provide
25	>	the corresponding part of the job script
26	>	('ws' stands for 'write script').
27		"""
28
29	<	tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
30	<	params = {'tmpDir':tmpDir}
31	<	return  params
32	<
33	<	def loggingInfo(self, id):
34	<	""" return logging info about job nj """
35	<	cmd = 'something'
36	<	#cmd_out = runCommand(cmd)
37	<	return ''
29	>	def __init__(self):
30	>	SchedulerLocal.__init__(self,"CONDOR")
31	>	self.datasetPath   = None
32	>	self.selectNoInput = None
33	>	self.OSBsize = None
34	>
35	>	self.environment_unique_identifier = None
36	>	return
37	>
38	>
39	>	def configure(self, cfg_params):
40	>	"""
41	>	Configure the scheduler with the config settings from the user
42	>	"""
43	>
44	>	SchedulerLocal.configure(self, cfg_params)
45	>
46	>	self.proxyValid=0
47	>	self.dontCheckProxy=int(cfg_params.get("GRID.dont_check_proxy",0))
48	>	self.space_token = cfg_params.get("USER.space_token",None)
49	>	try:
50	>	self.proxyServer = Downloader("http://cmsdoc.cern.ch/cms/LCG/crab/config/").config("myproxy_server.conf")
51	>	self.proxyServer = self.proxyServer.strip()
52	>	if self.proxyServer is None:
53	>	raise CrabException("myproxy_server.conf retrieved but empty")
54	>	except Exception, e:
55	>	common.logger.info("Problem setting myproxy server endpoint: using myproxy.cern.ch")
56	>	common.logger.debug(e)
57	>	self.proxyServer= 'myproxy.cern.ch'
58	>	self.group = cfg_params.get("GRID.group", None)
59	>	self.role = cfg_params.get("GRID.role", None)
60	>	self.VO = cfg_params.get('GRID.virtual_organization','cms')
61	>
62	>	try:
63	>	tmp =  cfg_params['CMSSW.datasetpath']
64	>	if tmp.lower() == 'none':
65	>	self.datasetPath = None
66	>	self.selectNoInput = 1
67	>	else:
68	>	self.datasetPath = tmp
69	>	self.selectNoInput = 0
70	>	except KeyError:
71	>	msg = "Error: datasetpath not defined "
72	>	raise CrabException(msg)
73	>
74	>	self.checkProxy()
75	>
76	>	return
77	>
78	>	def envUniqueID(self):
79	>	taskHash = sha1(common._db.queryTask('name')).hexdigest()
80	>	id = "https://" + socket.gethostname() + '/' + taskHash + "/${NJob}"
81	>	return id
82	>
83	>	def sched_parameter(self, i, task):
84	>	"""
85	>	Return scheduler-specific parameters
86	>	"""
87	>	req = ''
88	>	if self.EDG_addJdlParam:
89	>	if self.EDG_addJdlParam[-1] == '':
90	>	self.EDG_addJdlParam = self.EDG_addJdlParam[:-1]
91	>	for p in self.EDG_addJdlParam:
92	>	req += p.strip()+';\n'
93	>
94	>	return req
95	>
96	>
97	>	def realSchedParams(self, cfg_params):
98	>	"""
99	>	Return dictionary with specific parameters, to use with real scheduler
100	>	"""
101	>
102	>	tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
103	>	tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
104	>	jobDir = common.work_space.jobDir()
105	>	params = {'tmpDir':tmpDir,
106	>	'jobDir':jobDir}
107	>	return params
108	>
109	>
110	>	def listMatch(self, seList, full):
111	>	"""
112	>	Check the compatibility of available resources
113	>	"""
114	>
115	>	return [True]
116	>
117	>
118	>	def decodeLogInfo(self, fileName):
119	>	"""
120	>	Parse logging info file and return main info
121	>	"""
122	>
123	>	import CondorGLoggingInfo
124	>	loggingInfo = CondorGLoggingInfo.CondorGLoggingInfo()
125	>	reason = loggingInfo.decodeReason(fileName)
126	>	return reason
127	>
128	>
129	>	def wsCopyOutput(self):
130	>	"""
131	>	Write a CopyResults part of a job script, e.g.
132	>	to copy produced output into a storage element.
133	>	"""
134	>	txt = self.wsCopyOutput_comm()
135	>	return txt
136	>
137	>
138	>	def wsExitFunc(self):
139	>	"""
140	>	Returns the part of the job script which runs prior to exit
141	>	"""
142	>
143	>	txt = '\n'
144	>	txt += '#\n'
145	>	txt += '# EXECUTE THIS FUNCTION BEFORE EXIT \n'
146	>	txt += '#\n\n'
147	>
148	>	txt += 'func_exit() { \n'
149	>	txt += self.wsExitFunc_common()
150	>
151	>	txt += '    tar zcvf ${out_files}.tgz  ${final_list}\n'
152	>	txt += '    cp  ${out_files}.tgz $_CONDOR_SCRATCH_DIR/\n'
153	>	txt += '    cp  crab_fjr_$NJob.xml $_CONDOR_SCRATCH_DIR/\n'
154	>
155	>	txt += '    exit $job_exit_code\n'
156	>	txt += '}\n'
157	>
158	>	return txt
159	>
160	>	def wsInitialEnvironment(self):
161	>	"""
162	>	Returns part of a job script which does scheduler-specific work.
163	>	"""
164	>
165	>	txt  = '\n# Written by SchedulerCondor::wsInitialEnvironment\n'
166	>	txt += 'echo "Beginning environment"\n'
167	>	txt += 'printenv | sort\n'
168	>
169	>	txt += 'middleware='+self.name()+' \n'
170	>	txt += 'if [ -e /opt/d-cache/srm/bin ]; then\n'
171	>	txt += '  export PATH=${PATH}:/opt/d-cache/srm/bin\n'
172	>	txt += 'fi\n'
173	>
174	>	txt += """
175	>	if [ $_CONDOR_SCRATCH_DIR ] && [ -d $_CONDOR_SCRATCH_DIR ]; then
176	>	echo "cd to Condor scratch directory: $_CONDOR_SCRATCH_DIR"
177	>	if [ -e ../default.tgz ] ;then
178	>	echo "Found ISB in parent directory (Local Condor)"
179	>	cp ../default.tgz $_CONDOR_SCRATCH_DIR
180	>	fi
181	>	cd $_CONDOR_SCRATCH_DIR
182	>	fi
183	>	"""
184	>
185	>	return txt
186	>
187	>
188	>	def sched_fix_parameter(self):
189	>	"""
190	>	Returns string with requirements and scheduler-specific parameters
191	>	"""
192	>
193	>	if self.EDG_requirements:
194	>	req = self.EDG_requirements
195	>	taskReq = {'commonRequirements':req}
196	>	common._db.updateTask_(taskReq)

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines