CRAB/python/Creator.py

from Actor import Actor
from WorkSpace import WorkSpace
from JobList import JobList
from ScriptWriter import ScriptWriter
from Scheduler import Scheduler
from crab_logger import Logger
from crab_exceptions import *
from crab_util import *
import common

import os, string, math
import time

class Creator(Actor):
    def __init__(self, job_type_name, cfg_params, ncjobs, skip_blocks=False,isNew=True,first_jobID=0):
        self.job_type_name = job_type_name
        self.job_type = None
        self.cfg_params = cfg_params
        self.total_njobs = 0
        self.total_number_of_events = 0
        self.job_number_of_events = 0
        self.first_event = 0
        self.jobParamsList=[]
        self.first_jobID = first_jobID 
        self.isNew = isNew

        self.createJobTypeObject(ncjobs,skip_blocks,self.isNew)
        common.logger.debug(5, __name__+": JobType "+self.job_type.name()+" created")


        self.total_njobs = self.job_type.numberOfJobs();
        common.logger.debug(5, __name__+": total # of jobs = "+`self.total_njobs`)

        self.ncjobs = ncjobs
        if ncjobs == 'all' : self.ncjobs = self.total_njobs
        if ncjobs > self.total_njobs : self.ncjobs = self.total_njobs
        
        self.job_type_name = self.job_type.name()
 
        common.logger.debug(5, "Creator constructor finished")
        return

    def writeJobsSpecsToDB(self, first_jobID=0):
        """
        Write firstEvent and maxEvents in the DB for future use
        """

        self.job_type.split(self.jobParamsList,first_jobID)
        return

    def nJobs(self):
        return self.total_njobs
    

    def nJobsL(self):
        jobsL=[]
        for i in range(self.total_njobs):
            jobsL.append(self.first_jobID+i+1)   
        return jobsL

    def createJobTypeObject(self,ncjobs,skip_blocks,isNew):
        file_name = 'cms_'+ string.lower(self.job_type_name)
        klass_name = string.capitalize(self.job_type_name)

        try:
            klass = importName(file_name, klass_name)
        except KeyError:
            msg = 'No `class '+klass_name+'` found in file `'+file_name+'.py`'
            raise CrabException(msg)
        except ImportError, e:
            msg = 'Cannot create job type '+self.job_type_name
            msg += ' (file: '+file_name+', class '+klass_name+'):\n'
            msg += str(e)
            raise CrabException(msg)

        self.job_type = klass(self.cfg_params,ncjobs,skip_blocks,isNew)
        return

    def jobType(self):
        return self.job_type

    def run(self):
        """
        The main method of the class.
        """

        common.logger.debug(5, "Creator::run() called")
        start = time.time()
        # Instantiate ScriptWriter
        script_writer = ScriptWriter(self.cfg_params,'crab_template.sh')

        # Loop over jobs
        argsList = []
        njc = 0
        listID=[]
        listField=[]
        listRunField=[]
        run_jobToSave = {'status' :'C', \
                         'statusScheduler' : 'Created'  }
        for nj in range(self.total_njobs):
            nj=nj+self.first_jobID
            output=[]
            if njc == self.ncjobs : break

            common.logger.debug(1,"Creating job # "+`(nj+1)`)
            listRunField.append(run_jobToSave)

            # Prepare configuration file

            output.append('out_files_'+str(nj+1)+'.tgz')
            job_ToSave={'outputFiles': output}
            listField.append(job_ToSave)             

            listID.append(nj+1)
            njc = njc + 1
            pass
 
       # ## Not clear why here.. DS
       # self.job_type.setArgsList()
        common._db.updateRunJob_(listID , listRunField ) 
        common._db.updateJob_(listID, listField ) 

        if self.isNew:
            # Create script (sh)
            script_writer.modifyTemplateScript()
       
            # SL This should be a list, rather than a string!
            concString = ','
            inSand=''
            if len(self.job_type.inputSandbox(1)): 
                inSand +=   concString.join(self.job_type.inputSandbox(1)) 
            # Sandbox, Start Dir , outputDir
            param = {'globalSandbox': inSand , 'startDirectory': common.work_space.cwdDir() , 'outputDirectory': common.work_space.resDir() }
            common._db.updateTask_(param) 
            # define requirement
            common.scheduler.sched_fix_parameter()
        common.scheduler.declare(listID) 
       
        stop = time.time()
        common.logger.message('Creating '+str(self.total_njobs)+' jobs, please wait...')

        stop = time.time()
        common.logger.debug(2, "Creation Time: "+str(stop - start))
        common.logger.write("Creation Time: "+str(stop - start))

        msg = '\nTotal of %d jobs created'%njc
        if njc != self.ncjobs: msg = msg + ' from %d requested'%self.ncjobs
        msg = msg + '.\n'
        common.logger.message(msg)
        
        return
Revision:	1.71
Committed:	Thu Sep 25 11:50:05 2008 UTC (16 years, 7 months ago) by spiga
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_2_5_1, CRAB_2_5_1_pre4, CRAB_2_5_1_pre3, CRAB_2_5_1_pre2, CRAB_2_5_1_pre1, CRAB_2_5_0, CRAB_2_5_0_pre7, CRAB_2_5_0_pre6, CRAB_2_5_0_pre5, CRAB_2_5_0_pre4, CRAB_2_5_0_pre3, CRAB_2_5_0_pre2, CRAB_2_5_0_pre1, CRAB_2_4_4, CRAB_2_4_4_pre6, CRAB_2_4_4_pre5, CRAB_2_4_4_pre4, CRAB_2_4_4_pre3, CRAB_2_4_4_pre2, CRAB_2_4_4_pre1, CRAB_2_4_3, CRAB_2_4_3_pre8, CRAB_2_4_3_pre7, CRAB_2_4_3_pre6, CRAB_2_4_3_pre5, CRAB_2_4_3_pre3, CRAB_2_4_3_pre2, CRAB_2_4_3_pre1, CRAB_2_4_2, CRAB_2_4_2_pre3, CRAB_2_4_2_pre2, CRAB_2_4_2_pre1, CRAB_2_4_1, CRAB_2_4_1_pre4, CRAB_2_4_1_pre3, CRAB_2_4_1_pre2, CRAB_2_4_1_pre1, CRAB_2_4_0_Tutorial, CRAB_2_4_0_Tutorial_pre1, CRAB_2_4_0, CRAB_2_4_0_pre9, CRAB_2_4_0_pre8, CRAB_2_4_0_pre7, CRAB_2_4_0_pre6, CRAB_2_4_0_pre5, CRAB_2_4_0_pre4, CRAB_2_4_0_pre3, CRAB_2_4_0_pre2
Changes since 1.70:	+0 -1 lines
Log Message:	scriptName filed not more needed in BL DB. Adapted CRAB code
#	User	Rev	Content
1	nsmirnov	1.1	from Actor import Actor
2			from WorkSpace import WorkSpace
3			from JobList import JobList
4			from ScriptWriter import ScriptWriter
5			from Scheduler import Scheduler
6			from crab_logger import Logger
7			from crab_exceptions import *
8			from crab_util import *
9			import common
10
11	slacapra	1.8	import os, string, math
12	slacapra	1.45	import time
13
14	nsmirnov	1.1	class Creator(Actor):
15	spiga	1.70	def __init__(self, job_type_name, cfg_params, ncjobs, skip_blocks=False,isNew=True,first_jobID=0):
16	nsmirnov	1.1	self.job_type_name = job_type_name
17	corvo	1.30	self.job_type = None
18	nsmirnov	1.1	self.cfg_params = cfg_params
19	nsmirnov	1.2	self.total_njobs = 0
20			self.total_number_of_events = 0
21			self.job_number_of_events = 0
22	slacapra	1.7	self.first_event = 0
23	gutsche	1.38	self.jobParamsList=[]
24	spiga	1.70	self.first_jobID = first_jobID
25			self.isNew = isNew
26
27			self.createJobTypeObject(ncjobs,skip_blocks,self.isNew)
28	nsmirnov	1.3	common.logger.debug(5, __name__+": JobType "+self.job_type.name()+" created")
29	nsmirnov	1.1
30
31	gutsche	1.38	self.total_njobs = self.job_type.numberOfJobs();
32	nsmirnov	1.3	common.logger.debug(5, __name__+": total # of jobs = "+`self.total_njobs`)
33	nsmirnov	1.1
34			self.ncjobs = ncjobs
35			if ncjobs == 'all' : self.ncjobs = self.total_njobs
36			if ncjobs > self.total_njobs : self.ncjobs = self.total_njobs
37	spiga	1.11
38	spiga	1.60	self.job_type_name = self.job_type.name()
39
40	nsmirnov	1.3	common.logger.debug(5, "Creator constructor finished")
41	nsmirnov	1.1	return
42
43	spiga	1.70	def writeJobsSpecsToDB(self, first_jobID=0):
44	slacapra	1.7	"""
45			Write firstEvent and maxEvents in the DB for future use
46			"""
47
48	spiga	1.70	self.job_type.split(self.jobParamsList,first_jobID)
49	nsmirnov	1.2	return
50
51	nsmirnov	1.1	def nJobs(self):
52			return self.total_njobs
53
54	spiga	1.61
55			def nJobsL(self):
56			jobsL=[]
57			for i in range(self.total_njobs):
58	spiga	1.70	jobsL.append(self.first_jobID+i+1)
59	spiga	1.61	return jobsL
60
61	spiga	1.70	def createJobTypeObject(self,ncjobs,skip_blocks,isNew):
62	nsmirnov	1.1	file_name = 'cms_'+ string.lower(self.job_type_name)
63			klass_name = string.capitalize(self.job_type_name)
64
65			try:
66			klass = importName(file_name, klass_name)
67			except KeyError:
68			msg = 'No `class '+klass_name+'` found in file `'+file_name+'.py`'
69			raise CrabException(msg)
70			except ImportError, e:
71			msg = 'Cannot create job type '+self.job_type_name
72			msg += ' (file: '+file_name+', class '+klass_name+'):\n'
73			msg += str(e)
74			raise CrabException(msg)
75
76	spiga	1.70	self.job_type = klass(self.cfg_params,ncjobs,skip_blocks,isNew)
77	nsmirnov	1.1	return
78	nsmirnov	1.5
79			def jobType(self):
80			return self.job_type
81	nsmirnov	1.1
82			def run(self):
83	nsmirnov	1.4	"""
84			The main method of the class.
85			"""
86	nsmirnov	1.1
87	nsmirnov	1.3	common.logger.debug(5, "Creator::run() called")
88	slacapra	1.45	start = time.time()
89	nsmirnov	1.1	# Instantiate ScriptWriter
90	spiga	1.69	script_writer = ScriptWriter(self.cfg_params,'crab_template.sh')
91	nsmirnov	1.1
92			# Loop over jobs
93	slacapra	1.44	argsList = []
94	nsmirnov	1.1	njc = 0
95	spiga	1.60	listID=[]
96			listField=[]
97	spiga	1.63	listRunField=[]
98	spiga	1.67	run_jobToSave = {'status' :'C', \
99			'statusScheduler' : 'Created' }
100	nsmirnov	1.4	for nj in range(self.total_njobs):
101	spiga	1.70	nj=nj+self.first_jobID
102	spiga	1.64	output=[]
103	nsmirnov	1.1	if njc == self.ncjobs : break
104
105	slacapra	1.44	common.logger.debug(1,"Creating job # "+`(nj+1)`)
106	spiga	1.63	listRunField.append(run_jobToSave)
107	nsmirnov	1.3
108	nsmirnov	1.1	# Prepare configuration file
109
110	spiga	1.64	output.append('out_files_'+str(nj+1)+'.tgz')
111			job_ToSave={'outputFiles': output}
112	spiga	1.60	listField.append(job_ToSave)
113	spiga	1.57
114	spiga	1.62	listID.append(nj+1)
115	nsmirnov	1.1	njc = njc + 1
116			pass
117	spiga	1.60
118			# ## Not clear why here.. DS
119			# self.job_type.setArgsList()
120	spiga	1.70	common._db.updateRunJob_(listID , listRunField )
121			common._db.updateJob_(listID, listField )
122	spiga	1.57
123	spiga	1.70	if self.isNew:
124			# Create script (sh)
125			script_writer.modifyTemplateScript()
126
127			# SL This should be a list, rather than a string!
128			concString = ','
129			inSand=''
130			if len(self.job_type.inputSandbox(1)):
131			inSand += concString.join(self.job_type.inputSandbox(1))
132			# Sandbox, Start Dir , outputDir
133			param = {'globalSandbox': inSand , 'startDirectory': common.work_space.cwdDir() , 'outputDirectory': common.work_space.resDir() }
134			common._db.updateTask_(param)
135			# define requirement
136			common.scheduler.sched_fix_parameter()
137			common.scheduler.declare(listID)
138	spiga	1.60
139			stop = time.time()
140	slacapra	1.44	common.logger.message('Creating '+str(self.total_njobs)+' jobs, please wait...')
141	fanzago	1.12
142	slacapra	1.45	stop = time.time()
143	slacapra	1.47	common.logger.debug(2, "Creation Time: "+str(stop - start))
144	slacapra	1.45	common.logger.write("Creation Time: "+str(stop - start))
145
146	nsmirnov	1.1	msg = '\nTotal of %d jobs created'%njc
147			if njc != self.ncjobs: msg = msg + ' from %d requested'%self.ncjobs
148			msg = msg + '.\n'
149			common.logger.message(msg)
150	slacapra	1.45
151	nsmirnov	1.1	return