CRAB/python/Creator.py

from Actor import Actor
from WorkSpace import WorkSpace
from JobList import JobList
from JobDB import JobDB
from ScriptWriter import ScriptWriter
from Scheduler import Scheduler
from crab_logger import Logger
from crab_exceptions import *
from crab_util import *
import common

import os, string

class Creator(Actor):
    def __init__(self, job_type_name, cfg_params, ncjobs):
        self.job_type_name = job_type_name
        self.job_type = None
        self.cfg_params = cfg_params
        self.total_njobs = 0
        self.ncjobs = 0                  # nb of jobs to be created
        self.total_number_of_events = 0
        self.job_number_of_events = 0
        
        #

        self.createJobTypeObject()
        common.logger.debug(5, __name__+": JobType "+self.job_type.name()+" created")

        self.job_type.prepareSteeringCards()
        common.logger.debug(5, __name__+": Steering cards prepared")

        self.defineTotalNumberOfJobs_()
        common.logger.debug(5, __name__+": total # of jobs = "+`self.total_njobs`)

        # Set number of jobs to be created

        self.ncjobs = ncjobs
        if ncjobs == 'all' : self.ncjobs = self.total_njobs
        if ncjobs > self.total_njobs : self.ncjobs = self.total_njobs

        #TODO: deprecated code, not needed,
        # will be eliminated when WorkSpace.saveConfiguration()
        # will be improved.
        #
        # Set/Save Job Type name

        jt_fname = common.work_space.shareDir() + 'jobtype'
        if os.path.exists(jt_fname):
            # Read stored job type name
            jt_file = open(jt_fname, 'r')
            jt = jt_file.read()
            if self.job_type_name:
                if ( jt != self.job_type_name+'\n' ):
                    msg = 'Job Type mismatch: requested <' + self.job_type_name
                    msg += '>, found <' + jt[:-1] + '>.'
                    raise CrabException(msg)
                pass
            else:
                self.job_type_name = jt[:-1]
                pass
            jt_file.close()
            pass
        else:
            # Save job type name
            jt_file = open(jt_fname, 'w')
            jt_file.write(self.job_type_name+'\n')
            jt_file.close()
            pass
        #end of deprecated code

        common.logger.debug(5, "Creator constructor finished")
        return

    def defineTotalNumberOfJobs_(self):
        """
        Calculates the total number of jobs to be created.
        """

        try:
            n = self.cfg_params['USER.total_number_of_events']
            if n == 'all': n = '-1'
            self.total_number_of_events = int(n)
        except KeyError:
            self.total_number_of_events = -1
            pass
        
        try:
            n = self.cfg_params['USER.job_number_of_events']
            self.job_number_of_events = int(n)
        except KeyError:
            msg = 'Warning. '
            msg += 'Number of events per job is not defined by user.\n'
            msg += 'Set to the total number of events.'
            common.logger.message(msg)
            self.job_number_of_events = self.total_number_of_events
            pass

        self.total_njobs = int((self.total_number_of_events-1)/self.job_number_of_events)+1
        return

    def nJobs(self):
        return self.total_njobs
    
    def createJobTypeObject(self):
        file_name = 'cms_'+ string.lower(self.job_type_name)
        klass_name = string.capitalize(self.job_type_name)

        try:
            klass = importName(file_name, klass_name)
        except KeyError:
            msg = 'No `class '+klass_name+'` found in file `'+file_name+'.py`'
            raise CrabException(msg)
        except ImportError, e:
            msg = 'Cannot create job type '+self.job_type_name
            msg += ' (file: '+file_name+', class '+klass_name+'):\n'
            msg += str(e)
            raise CrabException(msg)

        self.job_type = klass(self.cfg_params)
        return

    def jobType(self):
        return self.job_type

    def run(self):
        """
        The main method of the class.
        """

        common.logger.debug(5, "Creator::run() called")

        # Instantiate ScriptWriter

        script_writer = ScriptWriter('crab_template.sh')

        # Loop over jobs

        njc = 0
        for nj in range(self.total_njobs):
            if njc == self.ncjobs : break
            st = common.jobDB.status(nj)
            if st != 'X': continue

            common.logger.message("Creating job # "+`(nj+1)`)

            # Prepare configuration file

            self.job_type.modifySteeringCards(nj)

            # Create JDL
            # Maybe, it worths to move this call into Submitter,
            # i.e. to create scheduler-specific file at submission time ?

            common.scheduler.createJDL(nj)

            # Create script

            script_writer.modifyTemplateScript(nj)
            os.chmod(common.job_list[nj].scriptFilename(), 0744)

            common.jobDB.setStatus(nj, 'C')
            njc = njc + 1
            pass

        ####
        
        common.jobDB.save()

        msg = '\nTotal of %d jobs created'%njc
        if njc != self.ncjobs: msg = msg + ' from %d requested'%self.ncjobs
        msg = msg + '.\n'
        common.logger.message(msg)
        return
Revision:	1.6
Committed:	Thu Jun 30 10:35:27 2005 UTC (19 years, 10 months ago) by nsmirnov
Content type:	text/x-python
Branch:	MAIN
Changes since 1.5:	+1 -1 lines
Log Message:	more flexible syntax for job range
#	User	Rev	Content
1	nsmirnov	1.1	from Actor import Actor
2			from WorkSpace import WorkSpace
3			from JobList import JobList
4			from JobDB import JobDB
5			from ScriptWriter import ScriptWriter
6			from Scheduler import Scheduler
7			from crab_logger import Logger
8			from crab_exceptions import *
9			from crab_util import *
10			import common
11
12			import os, string
13
14			class Creator(Actor):
15			def __init__(self, job_type_name, cfg_params, ncjobs):
16			self.job_type_name = job_type_name
17			self.job_type = None
18			self.cfg_params = cfg_params
19	nsmirnov	1.2	self.total_njobs = 0
20			self.ncjobs = 0 # nb of jobs to be created
21			self.total_number_of_events = 0
22			self.job_number_of_events = 0
23
24	nsmirnov	1.1	#
25
26			self.createJobTypeObject()
27	nsmirnov	1.3	common.logger.debug(5, __name__+": JobType "+self.job_type.name()+" created")
28	nsmirnov	1.1
29			self.job_type.prepareSteeringCards()
30	nsmirnov	1.3	common.logger.debug(5, __name__+": Steering cards prepared")
31	nsmirnov	1.1
32	nsmirnov	1.2	self.defineTotalNumberOfJobs_()
33	nsmirnov	1.3	common.logger.debug(5, __name__+": total # of jobs = "+`self.total_njobs`)
34	nsmirnov	1.1
35			# Set number of jobs to be created
36
37			self.ncjobs = ncjobs
38			if ncjobs == 'all' : self.ncjobs = self.total_njobs
39			if ncjobs > self.total_njobs : self.ncjobs = self.total_njobs
40
41			#TODO: deprecated code, not needed,
42			# will be eliminated when WorkSpace.saveConfiguration()
43			# will be improved.
44			#
45			# Set/Save Job Type name
46
47			jt_fname = common.work_space.shareDir() + 'jobtype'
48			if os.path.exists(jt_fname):
49			# Read stored job type name
50			jt_file = open(jt_fname, 'r')
51			jt = jt_file.read()
52			if self.job_type_name:
53			if ( jt != self.job_type_name+'\n' ):
54			msg = 'Job Type mismatch: requested <' + self.job_type_name
55			msg += '>, found <' + jt[:-1] + '>.'
56			raise CrabException(msg)
57			pass
58			else:
59			self.job_type_name = jt[:-1]
60			pass
61			jt_file.close()
62			pass
63			else:
64			# Save job type name
65			jt_file = open(jt_fname, 'w')
66			jt_file.write(self.job_type_name+'\n')
67			jt_file.close()
68			pass
69			#end of deprecated code
70
71	nsmirnov	1.3	common.logger.debug(5, "Creator constructor finished")
72	nsmirnov	1.1	return
73
74	nsmirnov	1.2	def defineTotalNumberOfJobs_(self):
75			"""
76			Calculates the total number of jobs to be created.
77			"""
78
79			try:
80			n = self.cfg_params['USER.total_number_of_events']
81			if n == 'all': n = '-1'
82			self.total_number_of_events = int(n)
83			except KeyError:
84			self.total_number_of_events = -1
85			pass
86
87			try:
88			n = self.cfg_params['USER.job_number_of_events']
89			self.job_number_of_events = int(n)
90			except KeyError:
91			msg = 'Warning. '
92			msg += 'Number of events per job is not defined by user.\n'
93			msg += 'Set to the total number of events.'
94			common.logger.message(msg)
95			self.job_number_of_events = self.total_number_of_events
96			pass
97
98			self.total_njobs = int((self.total_number_of_events-1)/self.job_number_of_events)+1
99			return
100
101	nsmirnov	1.1	def nJobs(self):
102			return self.total_njobs
103
104			def createJobTypeObject(self):
105			file_name = 'cms_'+ string.lower(self.job_type_name)
106			klass_name = string.capitalize(self.job_type_name)
107
108			try:
109			klass = importName(file_name, klass_name)
110			except KeyError:
111			msg = 'No `class '+klass_name+'` found in file `'+file_name+'.py`'
112			raise CrabException(msg)
113			except ImportError, e:
114			msg = 'Cannot create job type '+self.job_type_name
115			msg += ' (file: '+file_name+', class '+klass_name+'):\n'
116			msg += str(e)
117			raise CrabException(msg)
118
119			self.job_type = klass(self.cfg_params)
120			return
121	nsmirnov	1.5
122			def jobType(self):
123			return self.job_type
124	nsmirnov	1.1
125			def run(self):
126	nsmirnov	1.4	"""
127			The main method of the class.
128			"""
129	nsmirnov	1.1
130	nsmirnov	1.3	common.logger.debug(5, "Creator::run() called")
131
132	nsmirnov	1.1	# Instantiate ScriptWriter
133
134			script_writer = ScriptWriter('crab_template.sh')
135
136			# Loop over jobs
137
138			njc = 0
139	nsmirnov	1.4	for nj in range(self.total_njobs):
140	nsmirnov	1.1	if njc == self.ncjobs : break
141	nsmirnov	1.4	st = common.jobDB.status(nj)
142	nsmirnov	1.1	if st != 'X': continue
143
144	nsmirnov	1.6	common.logger.message("Creating job # "+`(nj+1)`)
145	nsmirnov	1.3
146	nsmirnov	1.1	# Prepare configuration file
147
148	nsmirnov	1.4	self.job_type.modifySteeringCards(nj)
149	nsmirnov	1.1
150			# Create JDL
151	nsmirnov	1.4	# Maybe, it worths to move this call into Submitter,
152			# i.e. to create scheduler-specific file at submission time ?
153	nsmirnov	1.1
154	nsmirnov	1.4	common.scheduler.createJDL(nj)
155	nsmirnov	1.1
156			# Create script
157
158	nsmirnov	1.4	script_writer.modifyTemplateScript(nj)
159			os.chmod(common.job_list[nj].scriptFilename(), 0744)
160	nsmirnov	1.1
161	nsmirnov	1.4	common.jobDB.setStatus(nj, 'C')
162	nsmirnov	1.1	njc = njc + 1
163			pass
164
165			####
166
167			common.jobDB.save()
168
169			msg = '\nTotal of %d jobs created'%njc
170			if njc != self.ncjobs: msg = msg + ' from %d requested'%self.ncjobs
171			msg = msg + '.\n'
172			common.logger.message(msg)
173			return