CRAB/python/ScriptWriter.py

from WorkSpace import WorkSpace
from JobList import JobList
from Scheduler import Scheduler
from crab_exceptions import *
import common
import Scram
import string,os

class ScriptWriter:
    def __init__(self, cfg_params, template):
        # pattern -> action
        self.actions = {
            'title'                       : self.title_,
            'untar_software'              : self.untarSoftware_,
            'initial_environment'         : self.initialEnvironment_,
            'setup_scheduler_environment' : self.setupSchedulerEnvironment_,
            'setup_jobtype_environment'   : self.setupJobTypeEnvironment_,
            'copy_input'                  : self.copyInput_,
            'rewrite_cmssw_cfg'           : self.rewriteCMSSWcfg_,
            'build_executable'            : self.buildExe_,
            'run_executable'              : self.runExe_,
            'rename_output'               : self.renameOutput_,
            'copy_output'                 : self.copyOutput_,
            'parse_report'                : self.parseReport_,
            'modify_report'               : self.modifyReport_,
            'func_exit'                   : self.func_exit_
            }

        if os.path.isfile("./"+template):
            self.template = "./"+template
        elif os.getenv('CRABDIR') and os.path.isfile(os.getenv('CRABDIR')+'/python/'+template):
            self.template = os.getenv('CRABDIR')+'/python/'+template
        else:
            raise CrabException("No crab_template.sh found!")
        self.nj = -1     # current job number

        try:
            self.scram = Scram.Scram(None)
            self.CMSSWversion = self.scram.getSWVersion()
            parts = self.CMSSWversion.split('_')
            self.CMSSW_major = int(parts[1])
            self.CMSSW_minor = int(parts[2])
            self.CMSSW_patch = int(parts[3])
        except:
            raise CrabException("Could not determine CMSSW version")
        self.debug_wrapper=''
        debug = int(cfg_params.get('USER.debug_wrapper',0))
        if debug==1 : self.debug_wrapper='--debug'

        self.scriptName = cfg_params.get('CRAB.jobtype').upper()+'.sh'

        self.pset = cfg_params.get('CMSSW.pset','None')

        return

    def setAction(self, pattern, action):
        self.actions[pattern] = action
        return

    def modifyTemplateScript(self):
        """
        Create a script from scratch.
        """

        tpl = open(self.template, 'r')

        wrapper_fullPath = common.work_space.jobDir()+self.scriptName
        script = open(wrapper_fullPath,'w')

        for line in tpl:
            if len(line) > 6 and line[:6] == '#CRAB ':
                act_str = string.strip(line[6:])
                try:
                    action = self.actions[act_str]
                except KeyError:
                    continue

                if action:
                    txt = action()
                    script.write(txt)
                    pass
                else:
                    script.write(line)
                pass
            else:
                script.write(line)
                pass
            pass

        script.close()
        tpl.close()
        os.chmod(wrapper_fullPath, 0744)
        return

    def title_(self):
        #txt = '# This script was generated by '+common.prog_name
        #txt += ' (version '+common.prog_version_str+').\n'
        txt = 'echo "This script was generated by '+common.prog_name
        txt += ' (version '+common.prog_version_str+')."'
        return txt


    ### FEDE ###

    def untarSoftware_(self):
        """
        Returns part of a job script which untar CMSSW software.
        """
        jbt = common.job_list.type()

        txt = jbt.wsUntarSoftware(self.nj)

        #txt += 'executable='+exe+'\n'
        return txt

    ###########################################

    def setupSchedulerEnvironment_(self):
        """
        Returns part of a job script which does scheduler-specific work.
        """
        txt = common.scheduler.wsSetupEnvironment()
        return txt

    def initialEnvironment_(self):
        """
        Returns part of a job script which does scheduler-specific work.
        """
        txt = common.scheduler.wsInitialEnvironment()
        return txt

    def setupJobTypeEnvironment_(self):
        """
        Returns part of a job script which does jobtype-specific work.
        """
        jbt = common.job_list.type()
        txt = jbt.wsSetupEnvironment(self.nj)
        return txt

    def buildExe_(self):
        """
        Returns part of a job script which builds the binary executable.
        """
        jbt = common.job_list.type()

        txt = jbt.wsBuildExe(self.nj)

        job = common.job_list[self.nj]
        exe = job.type().executableName()

        txt += 'executable='+exe+'\n'
        return txt

    def runExe_(self):
        """
        Returns part of a job script which executes the application.
        """
        job = common.job_list[self.nj]
        args = job.type().executableArgs()

        txt  = ''
        txt += 'CPU_INFOS=0 \n'
        # NO carriage return for this line #Fabio
        txt += '/usr/bin/time -f \"%U %S %P\" -o cpu_timing.txt '
        txt += '$executable '+args
        txt += ' > executable.out 2>&1 \n'
        return txt

    def renameOutput_(self):
        """
        Returns part of a job script which renames output files.
        """
        jbt = common.job_list.type()
        txt = '\n'
        txt += jbt.wsRenameOutput(self.nj)
        return txt

    def copyInput_(self):
        """
        Returns part of a job script which copies input files from SE.
        """
        txt = common.scheduler.wsCopyInput()
        return txt

    def copyOutput_(self):
        """
        Returns part of a job script which copies output files to SE.
        """
        txt = common.scheduler.wsCopyOutput()
        return txt

    def parseReport_(self):
        """
        Returns part of a job script which parse the FrameworkJobReport.
        """
        jbt = common.job_list.type()
        txt = jbt.wsParseFJR()
        return txt

    def modifyReport_(self):
        """
        Returns part of a job script which modifies the FrameworkJobReport.
        """
        jbt = common.job_list.type()
        txt = jbt.wsModifyReport(self.nj)
        return txt

    def cleanEnv_(self):
        """
        In OSG environment this function removes the WORKING_DIR
        """
        jbt = common.job_list.type()
        txt = jbt.cleanEnv()
        return txt

    def func_exit_(self):
        """
        Returns part of a job script which does scheduler-specific
        output checks and management.
        """
        txt = common.scheduler.wsExitFunc()
        return txt

    def rewriteCMSSWcfg_(self):
        """
        Returns part of the script that runs writeCfg.py on the WN
        """
        txt=''
        if str(self.pset).upper() != 'NONE':
            txt += "# Rewrite cfg for this job\n"
            txt += "echo  $RUNTIME_AREA/writeCfg.py "+str(self.debug_wrapper)+" pset.py pset.py\n"
            txt += "python $RUNTIME_AREA/writeCfg.py "+str(self.debug_wrapper)+" pset.py pset.py\n"
            txt += """
        result=$?
        if [ $result -ne 0 ]; then
            echo "ERROR ==> problem re-writing config file"
            job_exit_code=10040
            func_exit
        fi\n
        """
            txt += "  \n"
            txt += "cat $RUNTIME_AREA/inputsReport.txt  \n"
        #    txt += "cat inputsReport.txt >> $RUNTIME_AREA/$repo \n"
        #    txt += "dumpStatus $RUNTIME_AREA/$repo\n"

        return txt
Revision:	1.52
Committed:	Tue Apr 23 11:38:09 2013 UTC (12 years ago) by belforte
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_2_9_1, CRAB_2_9_1_pre2, CRAB_2_9_1_pre1, CRAB_2_9_0, CRAB_2_9_0_pre2, CRAB_2_9_0_pre1, CRAB_2_8_8, CRAB_2_8_8_pre1, CRAB_2_8_7_patch3, CRAB_2_8_7_patch2, CRAB_2_8_7_patch1, CRAB_2_8_7, CRAB_2_8_7_pre2, HEAD
Changes since 1.51:	+1 -1 lines
Log Message:	initialize CPU time to 0, not -1, see https://savannah.cern.ch/bugs/index.php?101295
#	User	Rev	Content
1	nsmirnov	1.1	from WorkSpace import WorkSpace
2			from JobList import JobList
3			from Scheduler import Scheduler
4			from crab_exceptions import *
5			import common
6	ewv	1.21	import Scram
7	slacapra	1.6	import string,os
8	nsmirnov	1.1
9			class ScriptWriter:
10	ewv	1.32	def __init__(self, cfg_params, template):
11	nsmirnov	1.1	# pattern -> action
12			self.actions = {
13	nsmirnov	1.3	'title' : self.title_,
14	fanzago	1.23	'untar_software' : self.untarSoftware_,
15	ewv	1.32	'initial_environment' : self.initialEnvironment_,
16	nsmirnov	1.3	'setup_scheduler_environment' : self.setupSchedulerEnvironment_,
17			'setup_jobtype_environment' : self.setupJobTypeEnvironment_,
18	fanzago	1.8	'copy_input' : self.copyInput_,
19	ewv	1.21	'rewrite_cmssw_cfg' : self.rewriteCMSSWcfg_,
20	nsmirnov	1.3	'build_executable' : self.buildExe_,
21			'run_executable' : self.runExe_,
22			'rename_output' : self.renameOutput_,
23	fanzago	1.5	'copy_output' : self.copyOutput_,
24	spiga	1.30	'parse_report' : self.parseReport_,
25	fanzago	1.12	'modify_report' : self.modifyReport_,
26	spiga	1.27	'func_exit' : self.func_exit_
27	nsmirnov	1.1	}
28	ewv	1.21
29	slacapra	1.6	if os.path.isfile("./"+template):
30			self.template = "./"+template
31			elif os.getenv('CRABDIR') and os.path.isfile(os.getenv('CRABDIR')+'/python/'+template):
32			self.template = os.getenv('CRABDIR')+'/python/'+template
33			else:
34			raise CrabException("No crab_template.sh found!")
35	nsmirnov	1.2	self.nj = -1 # current job number
36	mcinquil	1.14
37	ewv	1.21	try:
38			self.scram = Scram.Scram(None)
39			self.CMSSWversion = self.scram.getSWVersion()
40			parts = self.CMSSWversion.split('_')
41			self.CMSSW_major = int(parts[1])
42			self.CMSSW_minor = int(parts[2])
43			self.CMSSW_patch = int(parts[3])
44			except:
45			raise CrabException("Could not determine CMSSW version")
46	spiga	1.42	self.debug_wrapper=''
47	fanzago	1.41	debug = int(cfg_params.get('USER.debug_wrapper',0))
48			if debug==1 : self.debug_wrapper='--debug'
49	spiga	1.37
50	ewv	1.45	self.scriptName = cfg_params.get('CRAB.jobtype').upper()+'.sh'
51
52	spiga	1.44	self.pset = cfg_params.get('CMSSW.pset','None')
53	spiga	1.37
54	nsmirnov	1.1	return
55
56			def setAction(self, pattern, action):
57			self.actions[pattern] = action
58			return
59	ewv	1.21
60	slacapra	1.10	def modifyTemplateScript(self):
61	nsmirnov	1.1	"""
62			Create a script from scratch.
63			"""
64	ewv	1.21
65	nsmirnov	1.1	tpl = open(self.template, 'r')
66	ewv	1.45
67	spiga	1.38	wrapper_fullPath = common.work_space.jobDir()+self.scriptName
68			script = open(wrapper_fullPath,'w')
69	nsmirnov	1.1
70			for line in tpl:
71			if len(line) > 6 and line[:6] == '#CRAB ':
72			act_str = string.strip(line[6:])
73			try:
74			action = self.actions[act_str]
75			except KeyError:
76			continue
77
78			if action:
79			txt = action()
80			script.write(txt)
81			pass
82			else:
83			script.write(line)
84			pass
85			else:
86			script.write(line)
87			pass
88			pass
89
90			script.close()
91			tpl.close()
92	spiga	1.39	os.chmod(wrapper_fullPath, 0744)
93	nsmirnov	1.1	return
94
95	nsmirnov	1.2	def title_(self):
96	fanzago	1.46	#txt = '# This script was generated by '+common.prog_name
97			#txt += ' (version '+common.prog_version_str+').\n'
98			txt = 'echo "This script was generated by '+common.prog_name
99			txt += ' (version '+common.prog_version_str+')."'
100	nsmirnov	1.1	return txt
101	ewv	1.26
102	fanzago	1.23
103			### FEDE ###
104	ewv	1.26
105	fanzago	1.23	def untarSoftware_(self):
106			"""
107			Returns part of a job script which untar CMSSW software.
108			"""
109			jbt = common.job_list.type()
110
111			txt = jbt.wsUntarSoftware(self.nj)
112
113			#txt += 'executable='+exe+'\n'
114			return txt
115	ewv	1.26
116	fanzago	1.23	###########################################
117	ewv	1.21
118	nsmirnov	1.3	def setupSchedulerEnvironment_(self):
119			"""
120			Returns part of a job script which does scheduler-specific work.
121			"""
122			txt = common.scheduler.wsSetupEnvironment()
123			return txt
124	nsmirnov	1.1
125	ewv	1.32	def initialEnvironment_(self):
126			"""
127			Returns part of a job script which does scheduler-specific work.
128			"""
129			txt = common.scheduler.wsInitialEnvironment()
130			return txt
131
132	nsmirnov	1.3	def setupJobTypeEnvironment_(self):
133	nsmirnov	1.1	"""
134	fanzago	1.7	Returns part of a job script which does jobtype-specific work.
135	nsmirnov	1.1	"""
136	nsmirnov	1.3	jbt = common.job_list.type()
137			txt = jbt.wsSetupEnvironment(self.nj)
138			return txt
139	ewv	1.21
140	nsmirnov	1.2	def buildExe_(self):
141	nsmirnov	1.1	"""
142	nsmirnov	1.2	Returns part of a job script which builds the binary executable.
143	nsmirnov	1.1	"""
144	nsmirnov	1.2	jbt = common.job_list.type()
145	nsmirnov	1.1
146	nsmirnov	1.3	txt = jbt.wsBuildExe(self.nj)
147	nsmirnov	1.1
148	nsmirnov	1.2	job = common.job_list[self.nj]
149	nsmirnov	1.1	exe = job.type().executableName()
150
151	nsmirnov	1.2	txt += 'executable='+exe+'\n'
152			return txt
153
154			def runExe_(self):
155			"""
156			Returns part of a job script which executes the application.
157			"""
158	gutsche	1.9	job = common.job_list[self.nj]
159			args = job.type().executableArgs()
160	farinafa	1.34
161			txt = ''
162	belforte	1.52	txt += 'CPU_INFOS=0 \n'
163	ewv	1.45	# NO carriage return for this line #Fabio
164	farinafa	1.35	txt += '/usr/bin/time -f \"%U %S %P\" -o cpu_timing.txt '
165	belforte	1.51	txt += '$executable '+args
166			txt += ' > executable.out 2>&1 \n'
167	farinafa	1.34	return txt
168	nsmirnov	1.1
169	nsmirnov	1.3	def renameOutput_(self):
170			"""
171			Returns part of a job script which renames output files.
172			"""
173			jbt = common.job_list.type()
174			txt = '\n'
175			txt += jbt.wsRenameOutput(self.nj)
176			return txt
177	fanzago	1.5
178	fanzago	1.8	def copyInput_(self):
179			"""
180			Returns part of a job script which copies input files from SE.
181			"""
182			txt = common.scheduler.wsCopyInput()
183			return txt
184
185	fanzago	1.5	def copyOutput_(self):
186			"""
187			Returns part of a job script which copies output files to SE.
188			"""
189			txt = common.scheduler.wsCopyOutput()
190			return txt
191
192	spiga	1.30	def parseReport_(self):
193			"""
194			Returns part of a job script which parse the FrameworkJobReport.
195			"""
196			jbt = common.job_list.type()
197	spiga	1.31	txt = jbt.wsParseFJR()
198	spiga	1.30	return txt
199
200	fanzago	1.11	def modifyReport_(self):
201			"""
202			Returns part of a job script which modifies the FrameworkJobReport.
203			"""
204			jbt = common.job_list.type()
205	spiga	1.36	txt = jbt.wsModifyReport(self.nj)
206	fanzago	1.11	return txt
207	fanzago	1.12
208			def cleanEnv_(self):
209			"""
210			In OSG environment this function removes the WORKING_DIR
211			"""
212			jbt = common.job_list.type()
213			txt = jbt.cleanEnv()
214			return txt
215	mcinquil	1.14
216	spiga	1.27	def func_exit_(self):
217	mcinquil	1.14	"""
218	ewv	1.32	Returns part of a job script which does scheduler-specific
219	spiga	1.27	output checks and management.
220	mcinquil	1.14	"""
221	spiga	1.27	txt = common.scheduler.wsExitFunc()
222	mcinquil	1.14	return txt
223	spiga	1.22
224	ewv	1.21	def rewriteCMSSWcfg_(self):
225			"""
226			Returns part of the script that runs writeCfg.py on the WN
227			"""
228	ewv	1.45	txt=''
229			if str(self.pset).upper() != 'NONE':
230	spiga	1.44	txt += "# Rewrite cfg for this job\n"
231	ewv	1.45	txt += "echo $RUNTIME_AREA/writeCfg.py "+str(self.debug_wrapper)+" pset.py pset.py\n"
232			txt += "python $RUNTIME_AREA/writeCfg.py "+str(self.debug_wrapper)+" pset.py pset.py\n"
233	ewv	1.47	txt += """
234			result=$?
235			if [ $result -ne 0 ]; then
236			echo "ERROR ==> problem re-writing config file"
237			job_exit_code=10040
238			func_exit
239	spiga	1.48	fi\n
240	ewv	1.47	"""
241	spiga	1.48	txt += " \n"
242	spiga	1.50	txt += "cat $RUNTIME_AREA/inputsReport.txt \n"
243	spiga	1.49	# txt += "cat inputsReport.txt >> $RUNTIME_AREA/$repo \n"
244			# txt += "dumpStatus $RUNTIME_AREA/$repo\n"
245	ewv	1.21
246			return txt