CRAB/python/Creator.py

from Actor import Actor
from WorkSpace import WorkSpace
from JobList import JobList
from JobDB import JobDB
from ScriptWriter import ScriptWriter
from Scheduler import Scheduler
from crab_logger import Logger
from crab_exceptions import *
from crab_util import *
import common

import os, string

class Creator(Actor):
    def __init__(self, job_type_name, cfg_params, ncjobs):
        self.job_type_name = job_type_name
        self.job_type = None
        self.cfg_params = cfg_params
        self.total_njobs = 0
        self.ncjobs = 0                  # nb of jobs to be created
        self.total_number_of_events = 0
        self.job_number_of_events = 0
        
        #

        self.createJobTypeObject()

        self.job_type.prepareSteeringCards()

        self.defineTotalNumberOfJobs_()

        # Set number of jobs to be created

        self.ncjobs = ncjobs
        if ncjobs == 'all' : self.ncjobs = self.total_njobs
        if ncjobs > self.total_njobs : self.ncjobs = self.total_njobs

        # Create and initialize JobList

        common.job_list = JobList(self.total_njobs, self.job_type)

        common.job_list.setScriptNames(self.job_type_name+'.sh')
        common.job_list.setJDLNames(self.job_type_name+'.jdl')

        #TODO: Should be moved into JobType.
        # Probable use-case -- more than one cfg-file.
        # job_list.setCfgNames() -> job_list.addCfgNames()
        try:
            dataset = self.cfg_params['USER.dataset']
        except KeyError:
            msg = 'dataset must be defined in the section [USER]'
            raise CrabException(msg)
        common.job_list.setCfgNames(dataset+'.orcarc')

        #TODO: deprecated code, not needed,
        # will be eliminated when WorkSpace.saveConfiguration()
        # will be improved.
        #
        # Set/Save Job Type name

        jt_fname = common.work_space.shareDir() + 'jobtype'
        if os.path.exists(jt_fname):
            # Read stored job type name
            jt_file = open(jt_fname, 'r')
            jt = jt_file.read()
            if self.job_type_name:
                if ( jt != self.job_type_name+'\n' ):
                    msg = 'Job Type mismatch: requested <' + self.job_type_name
                    msg += '>, found <' + jt[:-1] + '>.'
                    raise CrabException(msg)
                pass
            else:
                self.job_type_name = jt[:-1]
                pass
            jt_file.close()
            pass
        else:
            # Save job type name
            jt_file = open(jt_fname, 'w')
            jt_file.write(self.job_type_name+'\n')
            jt_file.close()
            pass
        #end of deprecated code

        return

    def defineTotalNumberOfJobs_(self):
        """
        Calculates the total number of jobs to be created.
        """

        try:
            n = self.cfg_params['USER.total_number_of_events']
            if n == 'all': n = '-1'
            self.total_number_of_events = int(n)
        except KeyError:
            self.total_number_of_events = -1
            pass
        
        # if total number of events is not specified
        # then maybe JobType knows it.
        if self.total_number_of_events == -1:
            self.total_number_of_events = self.job_type.nEvents()
            if self.total_number_of_events == -1:
                msg = 'Cannot set total_number_of_events'
                raise CrabException(msg)
            pass
        
        try:
            n = self.cfg_params['USER.job_number_of_events']
            self.job_number_of_events = int(n)
        except KeyError:
            msg = 'Warning. '
            msg += 'Number of events per job is not defined by user.\n'
            msg += 'Set to the total number of events.'
            common.logger.message(msg)
            self.job_number_of_events = self.total_number_of_events
            pass

        self.total_njobs = int((self.total_number_of_events-1)/self.job_number_of_events)+1
        return

    def nJobs(self):
        return self.total_njobs
    
    def createJobTypeObject(self):
        file_name = 'cms_'+ string.lower(self.job_type_name)
        klass_name = string.capitalize(self.job_type_name)

        try:
            klass = importName(file_name, klass_name)
        except KeyError:
            msg = 'No `class '+klass_name+'` found in file `'+file_name+'.py`'
            raise CrabException(msg)
        except ImportError, e:
            msg = 'Cannot create job type '+self.job_type_name
            msg += ' (file: '+file_name+', class '+klass_name+'):\n'
            msg += str(e)
            raise CrabException(msg)

        self.job_type = klass(self.cfg_params)
        return
    

    def run(self):

        # Instantiate ScriptWriter

        script_writer = ScriptWriter('crab_template.sh')

        # Loop over jobs

        njc = 0
        for i in range(self.total_njobs):
            if njc == self.ncjobs : break
            st = common.jobDB.status(i)
            if st != 'X': continue

            # Prepare configuration file

            self.job_type.modifySteeringCards(i)

            # Create JDL

            common.scheduler.createJDL(i)

            # Create script

            script_writer.modifyTemplateScript(i)
            os.chmod(common.job_list[i].scriptFilename(), 0744)

            common.jobDB.setStatus(i, 'C')
            njc = njc + 1
            pass

        ####
        
        common.jobDB.save()

        msg = '\nTotal of %d jobs created'%njc
        if njc != self.ncjobs: msg = msg + ' from %d requested'%self.ncjobs
        msg = msg + '.\n'
        common.logger.message(msg)
        return
Revision:	1.2
Committed:	Tue Jun 7 07:21:04 2005 UTC (19 years, 10 months ago) by nsmirnov
Content type:	text/x-python
Branch:	MAIN
Changes since 1.1:	+42 -32 lines
Log Message:	pubdb.py renamed to PubDB.py
#	User	Rev	Content
1	nsmirnov	1.1	from Actor import Actor
2			from WorkSpace import WorkSpace
3			from JobList import JobList
4			from JobDB import JobDB
5			from ScriptWriter import ScriptWriter
6			from Scheduler import Scheduler
7			from crab_logger import Logger
8			from crab_exceptions import *
9			from crab_util import *
10			import common
11
12			import os, string
13
14			class Creator(Actor):
15			def __init__(self, job_type_name, cfg_params, ncjobs):
16			self.job_type_name = job_type_name
17			self.job_type = None
18			self.cfg_params = cfg_params
19	nsmirnov	1.2	self.total_njobs = 0
20			self.ncjobs = 0 # nb of jobs to be created
21			self.total_number_of_events = 0
22			self.job_number_of_events = 0
23
24	nsmirnov	1.1	#
25
26			self.createJobTypeObject()
27
28			self.job_type.prepareSteeringCards()
29
30	nsmirnov	1.2	self.defineTotalNumberOfJobs_()
31	nsmirnov	1.1
32			# Set number of jobs to be created
33
34			self.ncjobs = ncjobs
35			if ncjobs == 'all' : self.ncjobs = self.total_njobs
36			if ncjobs > self.total_njobs : self.ncjobs = self.total_njobs
37
38			# Create and initialize JobList
39
40			common.job_list = JobList(self.total_njobs, self.job_type)
41
42			common.job_list.setScriptNames(self.job_type_name+'.sh')
43			common.job_list.setJDLNames(self.job_type_name+'.jdl')
44
45			#TODO: Should be moved into JobType.
46			# Probable use-case -- more than one cfg-file.
47			# job_list.setCfgNames() -> job_list.addCfgNames()
48			try:
49			dataset = self.cfg_params['USER.dataset']
50			except KeyError:
51			msg = 'dataset must be defined in the section [USER]'
52			raise CrabException(msg)
53			common.job_list.setCfgNames(dataset+'.orcarc')
54
55			#TODO: deprecated code, not needed,
56			# will be eliminated when WorkSpace.saveConfiguration()
57			# will be improved.
58			#
59			# Set/Save Job Type name
60
61			jt_fname = common.work_space.shareDir() + 'jobtype'
62			if os.path.exists(jt_fname):
63			# Read stored job type name
64			jt_file = open(jt_fname, 'r')
65			jt = jt_file.read()
66			if self.job_type_name:
67			if ( jt != self.job_type_name+'\n' ):
68			msg = 'Job Type mismatch: requested <' + self.job_type_name
69			msg += '>, found <' + jt[:-1] + '>.'
70			raise CrabException(msg)
71			pass
72			else:
73			self.job_type_name = jt[:-1]
74			pass
75			jt_file.close()
76			pass
77			else:
78			# Save job type name
79			jt_file = open(jt_fname, 'w')
80			jt_file.write(self.job_type_name+'\n')
81			jt_file.close()
82			pass
83			#end of deprecated code
84
85			return
86
87	nsmirnov	1.2	def defineTotalNumberOfJobs_(self):
88			"""
89			Calculates the total number of jobs to be created.
90			"""
91
92			try:
93			n = self.cfg_params['USER.total_number_of_events']
94			if n == 'all': n = '-1'
95			self.total_number_of_events = int(n)
96			except KeyError:
97			self.total_number_of_events = -1
98			pass
99
100			# if total number of events is not specified
101			# then maybe JobType knows it.
102			if self.total_number_of_events == -1:
103			self.total_number_of_events = self.job_type.nEvents()
104			if self.total_number_of_events == -1:
105			msg = 'Cannot set total_number_of_events'
106			raise CrabException(msg)
107			pass
108
109			try:
110			n = self.cfg_params['USER.job_number_of_events']
111			self.job_number_of_events = int(n)
112			except KeyError:
113			msg = 'Warning. '
114			msg += 'Number of events per job is not defined by user.\n'
115			msg += 'Set to the total number of events.'
116			common.logger.message(msg)
117			self.job_number_of_events = self.total_number_of_events
118			pass
119
120			self.total_njobs = int((self.total_number_of_events-1)/self.job_number_of_events)+1
121			return
122
123	nsmirnov	1.1	def nJobs(self):
124			return self.total_njobs
125
126			def createJobTypeObject(self):
127			file_name = 'cms_'+ string.lower(self.job_type_name)
128			klass_name = string.capitalize(self.job_type_name)
129
130			try:
131			klass = importName(file_name, klass_name)
132			except KeyError:
133			msg = 'No `class '+klass_name+'` found in file `'+file_name+'.py`'
134			raise CrabException(msg)
135			except ImportError, e:
136			msg = 'Cannot create job type '+self.job_type_name
137			msg += ' (file: '+file_name+', class '+klass_name+'):\n'
138			msg += str(e)
139			raise CrabException(msg)
140
141			self.job_type = klass(self.cfg_params)
142			return
143
144
145			def run(self):
146
147			# Instantiate ScriptWriter
148
149			script_writer = ScriptWriter('crab_template.sh')
150
151			# Loop over jobs
152
153			njc = 0
154			for i in range(self.total_njobs):
155			if njc == self.ncjobs : break
156			st = common.jobDB.status(i)
157			if st != 'X': continue
158
159			# Prepare configuration file
160
161			self.job_type.modifySteeringCards(i)
162
163			# Create JDL
164
165			common.scheduler.createJDL(i)
166
167			# Create script
168
169			script_writer.modifyTemplateScript(i)
170			os.chmod(common.job_list[i].scriptFilename(), 0744)
171
172			common.jobDB.setStatus(i, 'C')
173			njc = njc + 1
174			pass
175
176			####
177
178			common.jobDB.save()
179
180			msg = '\nTotal of %d jobs created'%njc
181			if njc != self.ncjobs: msg = msg + ' from %d requested'%self.ncjobs
182			msg = msg + '.\n'
183			common.logger.message(msg)
184			return