CRAB/python/WorkSpace.py

from crab_exceptions import *
import logging
import common
import os, string, time
from crab_util import loadConfig

class WorkSpace:
    def __init__(self, top_dir, cfg_params):

        self._cwd_dir = os.getcwd()+'/'
        self._top_dir = top_dir    # top working directory

        self._user_out_dir = ''
        self._user_log_dir = ''
        if 'USER.outputdir' in cfg_params.keys():    
            self._user_out_dir = os.path.abspath(cfg_params['USER.outputdir'])
        if 'USER.logdir' in cfg_params.keys():    
            self._user_log_dir = os.path.abspath(cfg_params['USER.logdir'])
        #Matteo: Necessary to manage user ui_working_dir
        if 'USER.ui_working_dir' in cfg_params.keys():    
            self._top_dir = os.path.abspath(cfg_params['USER.ui_working_dir'])

       # self._res_dir = cfg_params.get("USER.outputdir", self._top_dir + '/res')     # dir to store job results
       # self._log_dir = ("USER.logdir",self._top_dir + '/log')    # log-directory
        self._log_dir = self._top_dir + '/log'    # log-directory
        self._job_dir = self._top_dir + '/job'     # job pars, scripts, jdl's
        self._res_dir = self._top_dir + '/res'     # dir to store job results
        self._share_dir = self._top_dir + '/share' # directory for common stuff

        #self._pathForTgz = string.split(top_dir, '/')[-1]
        self._pathForTgz = self._share_dir

        self.uuid = self.uuidGen()
 
    def uuidGen(self):
        import random

        letters = "abcdefghijklmnopqrstuvwxyz"
        numbers = "0123456789"
        genid_bits = random.sample(letters, 3) + random.sample(numbers, 3)
        random.shuffle(genid_bits)
        combination = ''.join(genid_bits)

        return combination   

    def create(self):
        # Matteo change in order to ban only "data" in "CMSSW" dir and 
        # not crash when short path is given    
        subpath = self._top_dir.split('CMSSW')
        if len(subpath)!=1 and ("data" in subpath[-1].split("/")):
            msg = 'Cannot run CRAB from "data" directory.\n'
            msg += 'please change direcotry\n'
            raise CrabException(msg)

        if not os.path.exists(self._top_dir):
            try:
                os.makedirs(self._top_dir)
            except OSError:
                msg = 'Cannot create '+str(self._top_dir) +' directory.\n'
                raise CrabException(msg)
            pass
        if not os.listdir(self._top_dir):
            os.mkdir(self._job_dir)
            os.mkdir(self._share_dir)
            os.mkdir(self._res_dir)
            os.mkdir(self._log_dir)
            pass

        # Some more check for _res_dir, since it can be user defined
        if self._user_out_dir != '':
            if not os.path.exists(self._user_out_dir):
                try: 
                    os.makedirs(self._user_out_dir)
                except:
                    msg = 'Cannot mkdir ' + self._user_out_dir + ' Check permission'
                    raise CrabException(msg)
            if os.listdir(self._user_out_dir):
                msg = self._user_out_dir + ' already exists and is not empty. Please remove it before create new task'
                raise CrabException(msg)
        # ditto for _log_dir
        if self._user_log_dir != '':
            if not os.path.exists(self._user_log_dir):
                try:
                    os.makedirs(self._user_log_dir)
                except:
                    msg = 'Cannot mkdir ' + self._user_log_dir + ' Check permission'
                    raise CrabException(msg)
                pass 
            if os.listdir(self._user_log_dir):
                msg = self._user_log_dir + ' already exists and is not empty. Please remove it before create new task'
                raise CrabException(msg)
        return

    def delete(self):
        """
        delete the whole workspace without doing any test!!!
        """
        common.logger.delete()
        common.logger = None

        if os.path.exists(self._top_dir):
            import shutil
            try:
                shutil.rmtree(self._top_dir)
            except :
                msg = 'Problem removing whole workspace. Please remove the remaining files manually.'
                raise CrabException(msg)
        return

    def cwdDir(self):
        if (self._cwd_dir[-1] != '/'):
           self._cwd_dir = self._cwd_dir + '/'
        return self._cwd_dir

    def topDir(self):
        if (self._top_dir[-1] != '/'):
           self._top_dir = self._top_dir + '/'
        return self._top_dir 

    def logDir(self):
        if (self._log_dir[-1] != '/'):
           self._log_dir = self._log_dir + '/'
        return self._log_dir

    def jobDir(self):
        if (self._job_dir[-1] != '/'):
           self._job_dir = self._job_dir + '/'
        return self._job_dir

    def resDir(self):
        if (self._res_dir[-1] != '/'):
           self._res_dir = self._res_dir + '/'
        return self._res_dir

    def shareDir(self):
        if (self._share_dir[-1] != '/'):
           self._share_dir = self._share_dir + '/'
        return self._share_dir
        
    def pathForTgz(self):
        if (self._pathForTgz[-1] != '/'):
           self._pathForTgz = self._pathForTgz + '/'
        return self._pathForTgz
        
    def task_uuid(self):
        return self.uuid

    def setResDir(self, dir):
        self._res_dir = dir
        return

    def saveFileName(self):
        return self.shareDir() + common.prog_name + '.sav'

    def cfgFileName(self):
        return self.shareDir() + common.prog_name + '.cfg'

    def saveConfiguration(self, opts, pars):

        # Save options
        
        save_file = open(self.saveFileName(), 'w')

        for k in opts.keys():
            if opts[k] : save_file.write(k+'='+opts[k]+'\n')
            else       : save_file.write(k+'\n')
            pass
        
        save_file.close()

        # Save cfg-file

        secs={}
        for k in pars.keys():
            sec, value = string.split(k,".")
            if sec not in secs.keys():
                secs[sec]={}
            secs[sec][value]=pars[k]
            pass
        
        config_file = open(self.cfgFileName(), 'w')
        for k in secs.keys():
            config_file.write('['+k+']\n')
            for v in secs[k].keys():
                if v!='debug': config_file.write(v+'='+secs[k][v]+'\n')
            else : config_file.write('\n')
        config_file.close()

        return

    def addToSavedCfg(self, opt):
        pars={}
        pars = loadConfig(self.shareDir() + common.prog_name + '.cfg', pars)
        pars[opt.keys()[0]]=opt.values()[0]

        secs={}
        for k in pars.keys():
            sec, value = string.split(k,".")
            if sec not in secs.keys():
                secs[sec]={}
            secs[sec][value]=pars[k]
            pass

        config_file = open(self.cfgFileName(), 'w')
        for k in secs.keys():
            config_file.write('['+k+']\n')
            for v in secs[k].keys():
                if v!='debug': config_file.write(v+'='+secs[k][v]+'\n')
            else : config_file.write('\n')
        config_file.close()

        return
    def loadSavedOptions(self):
        
        # Open save-file

        try:
            save_file = open(self.saveFileName(), 'r')
        except IOError, e:
            msg = 'Misconfigured continuation directory:\n'
            msg += str(e)
            raise CrabException(msg)

        # Read saved options

        save_opts = {}
        for line in save_file:
            line = line[:-1]  # drop '\n'
            try:
                (k, v) = string.split(line, '=')
            except:
                k=line
                v=''
                pass
            save_opts[k] = v
            pass
        
        save_file.close()
        return save_opts
    
Revision:	1.41
Committed:	Thu Jun 10 12:09:55 2010 UTC (14 years, 10 months ago) by spiga
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_2_7_8_patch2, CRAB_2_7_8_patch2_pre1, CRAB_2_7_8_patch1, CRAB_2_7_8_patch1_pre1, CRAB_2_7_8, CRAB_2_7_8_pre3, CRAB_2_7_8_pre2, CRAB_2_7_8_dash3, CRAB_2_7_8_dash2, CRAB_2_7_8_dash, CRAB_2_7_7_patch1, CRAB_2_7_7_patch1_pre1, CRAB_2_7_8_pre1, CRAB_2_7_7, CRAB_2_7_7_pre2, CRAB_2_7_7_pre1, CRAB_2_7_6_patch1, CRAB_2_7_6, CRAB_2_7_6_pre1, CRAB_2_7_5_patch1, CRAB_2_7_5, CRAB_2_7_5_pre3, CRAB_2_7_5_pre2, CRAB_2_7_5_pre1, CRAB_2_7_4_patch1, CRAB_2_7_4, CRAB_2_7_4_pre6, CRAB_2_7_4_pre5, CRAB_2_7_4_pre4, CRAB_2_7_4_pre3, CRAB_2_7_4_pre2, CRAB_2_7_4_pre1, CRAB_2_7_3, CRAB_2_7_3_pre3, CRAB_2_7_3_pre3_beta, CRAB_2_7_3_pre2
Changes since 1.40:	+22 -0 lines
Log Message:	I missed a piece of code.. :(
#	Content
1	from crab_exceptions import *
2	import logging
3	import common
4	import os, string, time
5	from crab_util import loadConfig
6
7	class WorkSpace:
8	def __init__(self, top_dir, cfg_params):
9
10	self._cwd_dir = os.getcwd()+'/'
11	self._top_dir = top_dir # top working directory
12
13	self._user_out_dir = ''
14	self._user_log_dir = ''
15	if 'USER.outputdir' in cfg_params.keys():
16	self._user_out_dir = os.path.abspath(cfg_params['USER.outputdir'])
17	if 'USER.logdir' in cfg_params.keys():
18	self._user_log_dir = os.path.abspath(cfg_params['USER.logdir'])
19	#Matteo: Necessary to manage user ui_working_dir
20	if 'USER.ui_working_dir' in cfg_params.keys():
21	self._top_dir = os.path.abspath(cfg_params['USER.ui_working_dir'])
22
23	# self._res_dir = cfg_params.get("USER.outputdir", self._top_dir + '/res') # dir to store job results
24	# self._log_dir = ("USER.logdir",self._top_dir + '/log') # log-directory
25	self._log_dir = self._top_dir + '/log' # log-directory
26	self._job_dir = self._top_dir + '/job' # job pars, scripts, jdl's
27	self._res_dir = self._top_dir + '/res' # dir to store job results
28	self._share_dir = self._top_dir + '/share' # directory for common stuff
29
30	#self._pathForTgz = string.split(top_dir, '/')[-1]
31	self._pathForTgz = self._share_dir
32
33	self.uuid = self.uuidGen()
34
35	def uuidGen(self):
36	import random
37
38	letters = "abcdefghijklmnopqrstuvwxyz"
39	numbers = "0123456789"
40	genid_bits = random.sample(letters, 3) + random.sample(numbers, 3)
41	random.shuffle(genid_bits)
42	combination = ''.join(genid_bits)
43
44	return combination
45
46	def create(self):
47	# Matteo change in order to ban only "data" in "CMSSW" dir and
48	# not crash when short path is given
49	subpath = self._top_dir.split('CMSSW')
50	if len(subpath)!=1 and ("data" in subpath[-1].split("/")):
51	msg = 'Cannot run CRAB from "data" directory.\n'
52	msg += 'please change direcotry\n'
53	raise CrabException(msg)
54
55	if not os.path.exists(self._top_dir):
56	try:
57	os.makedirs(self._top_dir)
58	except OSError:
59	msg = 'Cannot create '+str(self._top_dir) +' directory.\n'
60	raise CrabException(msg)
61	pass
62	if not os.listdir(self._top_dir):
63	os.mkdir(self._job_dir)
64	os.mkdir(self._share_dir)
65	os.mkdir(self._res_dir)
66	os.mkdir(self._log_dir)
67	pass
68
69	# Some more check for _res_dir, since it can be user defined
70	if self._user_out_dir != '':
71	if not os.path.exists(self._user_out_dir):
72	try:
73	os.makedirs(self._user_out_dir)
74	except:
75	msg = 'Cannot mkdir ' + self._user_out_dir + ' Check permission'
76	raise CrabException(msg)
77	if os.listdir(self._user_out_dir):
78	msg = self._user_out_dir + ' already exists and is not empty. Please remove it before create new task'
79	raise CrabException(msg)
80	# ditto for _log_dir
81	if self._user_log_dir != '':
82	if not os.path.exists(self._user_log_dir):
83	try:
84	os.makedirs(self._user_log_dir)
85	except:
86	msg = 'Cannot mkdir ' + self._user_log_dir + ' Check permission'
87	raise CrabException(msg)
88	pass
89	if os.listdir(self._user_log_dir):
90	msg = self._user_log_dir + ' already exists and is not empty. Please remove it before create new task'
91	raise CrabException(msg)
92	return
93
94	def delete(self):
95	"""
96	delete the whole workspace without doing any test!!!
97	"""
98	common.logger.delete()
99	common.logger = None
100
101	if os.path.exists(self._top_dir):
102	import shutil
103	try:
104	shutil.rmtree(self._top_dir)
105	except :
106	msg = 'Problem removing whole workspace. Please remove the remaining files manually.'
107	raise CrabException(msg)
108	return
109
110	def cwdDir(self):
111	if (self._cwd_dir[-1] != '/'):
112	self._cwd_dir = self._cwd_dir + '/'
113	return self._cwd_dir
114
115	def topDir(self):
116	if (self._top_dir[-1] != '/'):
117	self._top_dir = self._top_dir + '/'
118	return self._top_dir
119
120	def logDir(self):
121	if (self._log_dir[-1] != '/'):
122	self._log_dir = self._log_dir + '/'
123	return self._log_dir
124
125	def jobDir(self):
126	if (self._job_dir[-1] != '/'):
127	self._job_dir = self._job_dir + '/'
128	return self._job_dir
129
130	def resDir(self):
131	if (self._res_dir[-1] != '/'):
132	self._res_dir = self._res_dir + '/'
133	return self._res_dir
134
135	def shareDir(self):
136	if (self._share_dir[-1] != '/'):
137	self._share_dir = self._share_dir + '/'
138	return self._share_dir
139
140	def pathForTgz(self):
141	if (self._pathForTgz[-1] != '/'):
142	self._pathForTgz = self._pathForTgz + '/'
143	return self._pathForTgz
144
145	def task_uuid(self):
146	return self.uuid
147
148	def setResDir(self, dir):
149	self._res_dir = dir
150	return
151
152	def saveFileName(self):
153	return self.shareDir() + common.prog_name + '.sav'
154
155	def cfgFileName(self):
156	return self.shareDir() + common.prog_name + '.cfg'
157
158	def saveConfiguration(self, opts, pars):
159
160	# Save options
161
162	save_file = open(self.saveFileName(), 'w')
163
164	for k in opts.keys():
165	if opts[k] : save_file.write(k+'='+opts[k]+'\n')
166	else : save_file.write(k+'\n')
167	pass
168
169	save_file.close()
170
171	# Save cfg-file
172
173	secs={}
174	for k in pars.keys():
175	sec, value = string.split(k,".")
176	if sec not in secs.keys():
177	secs[sec]={}
178	secs[sec][value]=pars[k]
179	pass
180
181	config_file = open(self.cfgFileName(), 'w')
182	for k in secs.keys():
183	config_file.write('['+k+']\n')
184	for v in secs[k].keys():
185	if v!='debug': config_file.write(v+'='+secs[k][v]+'\n')
186	else : config_file.write('\n')
187	config_file.close()
188
189	return
190
191	def addToSavedCfg(self, opt):
192	pars={}
193	pars = loadConfig(self.shareDir() + common.prog_name + '.cfg', pars)
194	pars[opt.keys()[0]]=opt.values()[0]
195
196	secs={}
197	for k in pars.keys():
198	sec, value = string.split(k,".")
199	if sec not in secs.keys():
200	secs[sec]={}
201	secs[sec][value]=pars[k]
202	pass
203
204	config_file = open(self.cfgFileName(), 'w')
205	for k in secs.keys():
206	config_file.write('['+k+']\n')
207	for v in secs[k].keys():
208	if v!='debug': config_file.write(v+'='+secs[k][v]+'\n')
209	else : config_file.write('\n')
210	config_file.close()
211
212	return
213	def loadSavedOptions(self):
214
215	# Open save-file
216
217	try:
218	save_file = open(self.saveFileName(), 'r')
219	except IOError, e:
220	msg = 'Misconfigured continuation directory:\n'
221	msg += str(e)
222	raise CrabException(msg)
223
224	# Read saved options
225
226	save_opts = {}
227	for line in save_file:
228	line = line[:-1] # drop '\n'
229	try:
230	(k, v) = string.split(line, '=')
231	except:
232	k=line
233	v=''
234	pass
235	save_opts[k] = v
236	pass
237
238	save_file.close()
239	return save_opts
240