CRAB/python/WorkSpace.py

from crab_exceptions import *
import logging
import common
import os, string, time

class WorkSpace:
    def __init__(self, top_dir, cfg_params):

        self._cwd_dir = os.getcwd()+'/'
        self._top_dir = top_dir    # top working directory

        self._user_out_dir = ''
        self._user_log_dir = ''
        if 'USER.outputdir' in cfg_params.keys():    
            self._user_out_dir = os.path.abspath(cfg_params['USER.outputdir'])
        if 'USER.logdir' in cfg_params.keys():    
            self._user_log_dir = os.path.abspath(cfg_params['USER.logdir'])
        #Matteo: Necessary to manage user ui_working_dir
        if 'USER.ui_working_dir' in cfg_params.keys():    
            self._top_dir = os.path.abspath(cfg_params['USER.ui_working_dir'])

       # self._res_dir = cfg_params.get("USER.outputdir", self._top_dir + '/res')     # dir to store job results
       # self._log_dir = ("USER.logdir",self._top_dir + '/log')    # log-directory
        self._log_dir = self._top_dir + '/log'    # log-directory
        self._job_dir = self._top_dir + '/job'     # job pars, scripts, jdl's
        self._res_dir = self._top_dir + '/res'     # dir to store job results
        self._share_dir = self._top_dir + '/share' # directory for common stuff

        #self._pathForTgz = string.split(top_dir, '/')[-1]
        self._pathForTgz = self._share_dir

        self.uuid = self.uuidGen()
 
    def uuidGen(self):
        import random

        letters = "abcdefghijklmnopqrstuvwxyz"
        numbers = "0123456789"
        genid_bits = random.sample(letters, 3) + random.sample(numbers, 3)
        random.shuffle(genid_bits)
        combination = ''.join(genid_bits)

        return combination   

    def create(self):
        # Matteo change in order to ban only "data" in "CMSSW" dir and 
        # not crash when short path is given    
        subpath = self._top_dir.split('CMSSW')
        if len(subpath)!=1 and ("data" in subpath[-1].split("/")):
            msg = 'Cannot run CRAB from "data" directory.\n'
            msg += 'please change direcotry\n'
            raise CrabException(msg)

        if not os.path.exists(self._top_dir):
            try:
                os.mkdir(self._top_dir)
            except OSError:
                msg = 'Cannot create '+str(self._top_dir) +' directory.\n'
                raise CrabException(msg)
            pass
        if not os.listdir(self._top_dir):
            os.mkdir(self._job_dir)
            os.mkdir(self._share_dir)
            os.mkdir(self._res_dir)
            os.mkdir(self._log_dir)
            pass

        # Some more check for _res_dir, since it can be user defined
        if self._user_out_dir != '':
            if not os.path.exists(self._user_out_dir):
                try: 
                    os.mkdir(self._user_out_dir)
                except:
                    msg = 'Cannot mkdir ' + self._user_out_dir + ' Check permission'
                    raise CrabException(msg)
            if os.listdir(self._user_out_dir):
                msg = self._user_out_dir + ' already exists and is not empty. Please remove it before create new task'
                raise CrabException(msg)
        # ditto for _log_dir
        if self._user_log_dir != '':
            if not os.path.exists(self._user_log_dir):
                try:
                    os.mkdir(self._user_log_dir)
                except:
                    msg = 'Cannot mkdir ' + self._user_log_dir + ' Check permission'
                    raise CrabException(msg)
                pass 
            if os.listdir(self._user_log_dir):
                msg = self._user_log_dir + ' already exists and is not empty. Please remove it before create new task'
                raise CrabException(msg)
        return

    def delete(self):
        """
        delete the whole workspace without doing any test!!!
        """
        common.logger.delete()
        common.logger = None

        if os.path.exists(self._top_dir):
            import shutil
            try:
                shutil.rmtree(self._top_dir)
            except :
                msg = 'Problem removing whole workspace. Please remove the remaining files manually.'
                raise CrabException(msg)
        return

    def cwdDir(self):
        return self._cwd_dir + '/'

    def topDir(self):
        return self._top_dir + '/'

    def logDir(self):
        return self._log_dir + '/'

    def jobDir(self):
        return self._job_dir + '/'

    def resDir(self):
        return self._res_dir + '/'

    def shareDir(self):
        return self._share_dir + '/'
        
    def pathForTgz(self):
        return self._pathForTgz + '/'
        
    def task_uuid(self):
        return self.uuid

    def setResDir(self, dir):
        self._res_dir = dir
        return

    def saveFileName(self):
        return self.shareDir() + common.prog_name + '.sav'

    def cfgFileName(self):
        return self.shareDir() + common.prog_name + '.cfg'

    def saveConfiguration(self, opts, pars):

        # Save options
        
        save_file = open(self.saveFileName(), 'w')

        for k in opts.keys():
            if opts[k] : save_file.write(k+'='+opts[k]+'\n')
            else       : save_file.write(k+'\n')
            pass
        
        save_file.close()

        # Save cfg-file

        secs={}
        for k in pars.keys():
            sec, value = string.split(k,".")
            if sec not in secs.keys():
                secs[sec]={}
            secs[sec][value]=pars[k]
            pass
        
        config_file = open(self.cfgFileName(), 'w')
        for k in secs.keys():
            config_file.write('['+k+']\n')
            for v in secs[k].keys():
                config_file.write(v+'='+secs[k][v]+'\n')
            else : config_file.write('\n')
        config_file.close()

        return

    def loadSavedOptions(self):
        
        # Open save-file

        try:
            save_file = open(self.saveFileName(), 'r')
        except IOError, e:
            msg = 'Misconfigured continuation directory:\n'
            msg += str(e)
            raise CrabException(msg)

        # Read saved options

        save_opts = {}
        for line in save_file:
            line = line[:-1]  # drop '\n'
            try:
                (k, v) = string.split(line, '=')
            except:
                k=line
                v=''
                pass
            save_opts[k] = v
            pass
        
        save_file.close()
        return save_opts
    
Revision:	1.36
Committed:	Wed Aug 26 12:57:22 2009 UTC (15 years, 8 months ago) by farinafa
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_2_6_6_pre6, CRAB_2_6_6_pre5, CRAB_2_6_6_pre4, CRAB_2_6_6_pre3, CRAB_2_6_6_pre2, CRAB_2_6_6_check, CRAB_2_6_6, CRAB_2_6_6_pre1, CRAB_2_6_5, CRAB_2_6_5_pre1, CRAB_2_6_4, CRAB_2_6_4_pre1, CRAB_2_6_3_patch_2, CRAB_2_6_3_patch_2_pre2, CRAB_2_6_3_patch_2_pre1, CRAB_2_6_3_patch_1, CRAB_2_7_0_pre4, CRAB_2_7_0_pre3, CRAB_2_6_3, CRAB_2_6_3_pre5, CRAB_2_6_3_pre4, CRAB_2_6_3_pre3, CRAB_2_6_3_pre2, CRAB_2_7_0_pre2, CRAB_2_6_3_pre1, test_1, CRAB_2_7_0_pre1, CRAB_2_6_2, CRAB_2_6_2_pre2
Branch point for:	CRAB_2_6_X_br
Changes since 1.35:	+5 -2 lines
Log Message:	Bugfix on CleanerServer client-side. Added a missing catch clause in workspace delete method.
#	Content
1	from crab_exceptions import *
2	import logging
3	import common
4	import os, string, time
5
6	class WorkSpace:
7	def __init__(self, top_dir, cfg_params):
8
9	self._cwd_dir = os.getcwd()+'/'
10	self._top_dir = top_dir # top working directory
11
12	self._user_out_dir = ''
13	self._user_log_dir = ''
14	if 'USER.outputdir' in cfg_params.keys():
15	self._user_out_dir = os.path.abspath(cfg_params['USER.outputdir'])
16	if 'USER.logdir' in cfg_params.keys():
17	self._user_log_dir = os.path.abspath(cfg_params['USER.logdir'])
18	#Matteo: Necessary to manage user ui_working_dir
19	if 'USER.ui_working_dir' in cfg_params.keys():
20	self._top_dir = os.path.abspath(cfg_params['USER.ui_working_dir'])
21
22	# self._res_dir = cfg_params.get("USER.outputdir", self._top_dir + '/res') # dir to store job results
23	# self._log_dir = ("USER.logdir",self._top_dir + '/log') # log-directory
24	self._log_dir = self._top_dir + '/log' # log-directory
25	self._job_dir = self._top_dir + '/job' # job pars, scripts, jdl's
26	self._res_dir = self._top_dir + '/res' # dir to store job results
27	self._share_dir = self._top_dir + '/share' # directory for common stuff
28
29	#self._pathForTgz = string.split(top_dir, '/')[-1]
30	self._pathForTgz = self._share_dir
31
32	self.uuid = self.uuidGen()
33
34	def uuidGen(self):
35	import random
36
37	letters = "abcdefghijklmnopqrstuvwxyz"
38	numbers = "0123456789"
39	genid_bits = random.sample(letters, 3) + random.sample(numbers, 3)
40	random.shuffle(genid_bits)
41	combination = ''.join(genid_bits)
42
43	return combination
44
45	def create(self):
46	# Matteo change in order to ban only "data" in "CMSSW" dir and
47	# not crash when short path is given
48	subpath = self._top_dir.split('CMSSW')
49	if len(subpath)!=1 and ("data" in subpath[-1].split("/")):
50	msg = 'Cannot run CRAB from "data" directory.\n'
51	msg += 'please change direcotry\n'
52	raise CrabException(msg)
53
54	if not os.path.exists(self._top_dir):
55	try:
56	os.mkdir(self._top_dir)
57	except OSError:
58	msg = 'Cannot create '+str(self._top_dir) +' directory.\n'
59	raise CrabException(msg)
60	pass
61	if not os.listdir(self._top_dir):
62	os.mkdir(self._job_dir)
63	os.mkdir(self._share_dir)
64	os.mkdir(self._res_dir)
65	os.mkdir(self._log_dir)
66	pass
67
68	# Some more check for _res_dir, since it can be user defined
69	if self._user_out_dir != '':
70	if not os.path.exists(self._user_out_dir):
71	try:
72	os.mkdir(self._user_out_dir)
73	except:
74	msg = 'Cannot mkdir ' + self._user_out_dir + ' Check permission'
75	raise CrabException(msg)
76	if os.listdir(self._user_out_dir):
77	msg = self._user_out_dir + ' already exists and is not empty. Please remove it before create new task'
78	raise CrabException(msg)
79	# ditto for _log_dir
80	if self._user_log_dir != '':
81	if not os.path.exists(self._user_log_dir):
82	try:
83	os.mkdir(self._user_log_dir)
84	except:
85	msg = 'Cannot mkdir ' + self._user_log_dir + ' Check permission'
86	raise CrabException(msg)
87	pass
88	if os.listdir(self._user_log_dir):
89	msg = self._user_log_dir + ' already exists and is not empty. Please remove it before create new task'
90	raise CrabException(msg)
91	return
92
93	def delete(self):
94	"""
95	delete the whole workspace without doing any test!!!
96	"""
97	common.logger.delete()
98	common.logger = None
99
100	if os.path.exists(self._top_dir):
101	import shutil
102	try:
103	shutil.rmtree(self._top_dir)
104	except :
105	msg = 'Problem removing whole workspace. Please remove the remaining files manually.'
106	raise CrabException(msg)
107	return
108
109	def cwdDir(self):
110	return self._cwd_dir + '/'
111
112	def topDir(self):
113	return self._top_dir + '/'
114
115	def logDir(self):
116	return self._log_dir + '/'
117
118	def jobDir(self):
119	return self._job_dir + '/'
120
121	def resDir(self):
122	return self._res_dir + '/'
123
124	def shareDir(self):
125	return self._share_dir + '/'
126
127	def pathForTgz(self):
128	return self._pathForTgz + '/'
129
130	def task_uuid(self):
131	return self.uuid
132
133	def setResDir(self, dir):
134	self._res_dir = dir
135	return
136
137	def saveFileName(self):
138	return self.shareDir() + common.prog_name + '.sav'
139
140	def cfgFileName(self):
141	return self.shareDir() + common.prog_name + '.cfg'
142
143	def saveConfiguration(self, opts, pars):
144
145	# Save options
146
147	save_file = open(self.saveFileName(), 'w')
148
149	for k in opts.keys():
150	if opts[k] : save_file.write(k+'='+opts[k]+'\n')
151	else : save_file.write(k+'\n')
152	pass
153
154	save_file.close()
155
156	# Save cfg-file
157
158	secs={}
159	for k in pars.keys():
160	sec, value = string.split(k,".")
161	if sec not in secs.keys():
162	secs[sec]={}
163	secs[sec][value]=pars[k]
164	pass
165
166	config_file = open(self.cfgFileName(), 'w')
167	for k in secs.keys():
168	config_file.write('['+k+']\n')
169	for v in secs[k].keys():
170	config_file.write(v+'='+secs[k][v]+'\n')
171	else : config_file.write('\n')
172	config_file.close()
173
174	return
175
176	def loadSavedOptions(self):
177
178	# Open save-file
179
180	try:
181	save_file = open(self.saveFileName(), 'r')
182	except IOError, e:
183	msg = 'Misconfigured continuation directory:\n'
184	msg += str(e)
185	raise CrabException(msg)
186
187	# Read saved options
188
189	save_opts = {}
190	for line in save_file:
191	line = line[:-1] # drop '\n'
192	try:
193	(k, v) = string.split(line, '=')
194	except:
195	k=line
196	v=''
197	pass
198	save_opts[k] = v
199	pass
200
201	save_file.close()
202	return save_opts
203