[ViewVC] Annotation of: cvsroot/UserCode/MitProd/Processing/bin/remove

#!/usr/bin/env python
#---------------------------------------------------------------------------------------------------
# Script to remove a file.
#
# Author: C.Paus                                                                      (May 06, 2010)
#---------------------------------------------------------------------------------------------------
import os,sys,getopt,re,srm

def debugPrint(text):
    if debug:
        print ' DEBUG: ' + text

def execute(cmd,debug):
    if debug:
        print ' DEBUG: ' + cmd
    else:
        os.system(cmd)

def clean(file):
    if   re.search('dcap:',file):
        g = file.split('/')
        file = '/'.join(g[3:])
    debugPrint(' Cleaned: ' + file)

    return file


def exists(target):
    if   re.search('/castor/cern.ch/',target):
        debugPrint(' Identified a castor directory: ' + target)
        cmd = 'rfdir ' + target
    elif re.search('/pnfs/cmsaf.mit.edu/',target):
        debugPrint(' Identified a tier-2 directory: ' + target)
        cmd = 'ssh -x paus@cgate.mit.edu ls -1 ' + target + ' \>\& /dev/null'
    elif re.search('/mnt/hadoop/cms/store',target):
        debugPrint(' Identified a tier-2 hadoop directory: ' + target)
        target = srm.convertToUrl(target,debug)
        cmd = 'srmls ' + target + ' >& /dev/null'
    else:
        debugPrint(' Identified a normal directory: ' + target)
        cmd = 'ls -1 ' + target + '>& /dev/null'
            
    status = os.system(cmd)
    debugPrint(' Status: %d (on %s)' %(status,cmd))

    return (status == 0)

def remove(source):
    if   re.search('/castor/cern.ch/',source):
        debugPrint(' Identified castor file')
        cmd = "stager_rm -M " + source + "; nsrm " + source
    elif re.search('/pnfs/cmsaf.mit.edu/',source):
        debugPrint(' Identified tier-2 file')
        cmd = 'ssh -x paus@cgate.mit.edu rm -rf ' + source
    elif re.search('/mnt/hadoop/cms/store/user/paus',source):
        debugPrint(' Identified a tier-2 hadoop directory: ' + source)
        source = srm.convertToUrl(source,debug)
        cmd = 'srmrm ' + source + ' >& /dev/null'
    else:
        debugPrint(' Identified a normal directory')
        cmd = 'rm -rf ' + source

    # ready to perform the requested operation
    debugPrint('  -> removing with: ' + cmd)
    status = 0
    if exe == 1:
        status = os.system(cmd)
        if status != 0:
            print ' ERROR: remove returned error %d (on %s)'%(status,cmd)

    return status

def removeCatalog(source,debug):
    tmp = os.getpid()
    pid = "%d"%tmp
    # which catalog is this one in?
    catalogDir = '/home/cmsprod/catalog/local'
    if   re.search('/castor/cern.ch/',source):
        catalogDir = '/home/cmsprod/catalog/cern'
    elif re.search('/pnfs/cmsaf.mit.edu/',source) or \
         re.search('/mnt/hadoop/cms/store/user/paus',source):
        catalogDir = '/home/cmsprod/catalog/t2mit'
    # now get the dataset and the book
    f       = source.split('/')
    file    = f[-1]
    dataset = f[-2]
    book    = f[-4] + '/' + f[-3]

    dir     = catalogDir + '/' + book + '/' + dataset

    # now remove the particular file from the record
    cmd = 'cat ' + dir + '/RawFiles.?? | sort -u | grep -v ' + file + ' > /tmp/RawFiles.00.' + pid
    execute(cmd,debug)
    cmd = 'rm ' + dir + '/RawFiles.??'
    execute(cmd,debug)
    cmd = 'mv /tmp/RawFiles.00.' + pid + ' ' + dir + '/RawFiles.00'
    execute(cmd,debug)
    cmd = 'cat ' + dir + '/Files | grep -v ' + file + ' > /tmp/Files.' + pid
    execute(cmd,debug)
    cmd = 'mv /tmp/Files.' + pid + ' ' + dir + '/Files'
    execute(cmd,debug)

    return

#===================================================================================================
# Main starts here
#===================================================================================================
# Define string to explain usage of the script
usage =  "Usage: remove  <source>"
usage += "               --exe\n"
usage += "               --catalog\n"
usage += "               --debug\n"
usage += "               --help\n"

# Define the valid options which can be specified and check out the command line
valid = ['exe','catalog','debug','help']
try:
    opts, args = getopt.getopt(sys.argv[1:], "", valid)
except getopt.GetoptError, ex:
    print usage
    print str(ex)
    sys.exit(1)

# --------------------------------------------------------------------------------------------------
# Get all parameters for the production
# --------------------------------------------------------------------------------------------------
# Set defaults for each option
catalog = False
debug   = False
exe     = 0

# Read new values from the command line
for opt, arg in opts:
    #print ' OPT , ARG: ' + opt + ' ' + arg
    if   opt == '--help':
        print usage
        sys.exit(0)
    elif opt == '--debug':
        debug   = True
    elif opt == '--catalog':
        catalog = True
    elif opt == '--exe':
        exe     = 1

newArgv = []
for arg in sys.argv[1:]:
    #print ' ARG: ' + arg
    if arg[:2] == "--":
        continue
    else:
        newArgv.append(arg)

# Define source and target
source = clean(newArgv[0])

# Test whether the source exists
if   exists(source):
    debugPrint("\n Removing: " + source)
    remove(source)
elif not exists(source):
    print ' ERROR: the source ('+source+') does not exist.'

# Remove also the catalog entry
if catalog:
    removeCatalog(source,debug)
Revision:	1.5
Committed:	Tue Feb 28 11:54:36 2012 UTC (13 years, 2 months ago) by paus
Branch:	MAIN
CVS Tags:	Mit_028a, Mit_028, Mit_027a, Mit_027, Mit_026, Mit_025e, Mit_025d
Changes since 1.4:	+53 -7 lines
Log Message:	Last updates.
#	User	Rev	Content
1	paus	1.1	#!/usr/bin/env python
2			#---------------------------------------------------------------------------------------------------
3	paus	1.3	# Script to remove a file.
4	paus	1.1	#
5			# Author: C.Paus (May 06, 2010)
6			#---------------------------------------------------------------------------------------------------
7	paus	1.4	import os,sys,getopt,re,srm
8	paus	1.1
9			def debugPrint(text):
10	paus	1.4	if debug:
11	paus	1.1	print ' DEBUG: ' + text
12
13	paus	1.5	def execute(cmd,debug):
14			if debug:
15			print ' DEBUG: ' + cmd
16			else:
17			os.system(cmd)
18
19	paus	1.1	def clean(file):
20			if re.search('dcap:',file):
21			g = file.split('/')
22			file = '/'.join(g[3:])
23			debugPrint(' Cleaned: ' + file)
24
25			return file
26
27
28			def exists(target):
29			if re.search('/castor/cern.ch/',target):
30			debugPrint(' Identified a castor directory: ' + target)
31			cmd = 'rfdir ' + target
32			elif re.search('/pnfs/cmsaf.mit.edu/',target):
33			debugPrint(' Identified a tier-2 directory: ' + target)
34	paus	1.5	cmd = 'ssh -x paus@cgate.mit.edu ls -1 ' + target + ' \>\& /dev/null'
35	paus	1.4	elif re.search('/mnt/hadoop/cms/store',target):
36			debugPrint(' Identified a tier-2 hadoop directory: ' + target)
37			target = srm.convertToUrl(target,debug)
38			cmd = 'srmls ' + target + ' >& /dev/null'
39	paus	1.1	else:
40			debugPrint(' Identified a normal directory: ' + target)
41			cmd = 'ls -1 ' + target + '>& /dev/null'
42
43			status = os.system(cmd)
44	paus	1.4	debugPrint(' Status: %d (on %s)' %(status,cmd))
45	paus	1.1
46			return (status == 0)
47
48			def remove(source):
49			if re.search('/castor/cern.ch/',source):
50			debugPrint(' Identified castor file')
51			cmd = "stager_rm -M " + source + "; nsrm " + source
52			elif re.search('/pnfs/cmsaf.mit.edu/',source):
53			debugPrint(' Identified tier-2 file')
54	paus	1.5	cmd = 'ssh -x paus@cgate.mit.edu rm -rf ' + source
55	paus	1.4	elif re.search('/mnt/hadoop/cms/store/user/paus',source):
56			debugPrint(' Identified a tier-2 hadoop directory: ' + source)
57			source = srm.convertToUrl(source,debug)
58			cmd = 'srmrm ' + source + ' >& /dev/null'
59	paus	1.1	else:
60			debugPrint(' Identified a normal directory')
61	paus	1.2	cmd = 'rm -rf ' + source
62	paus	1.1
63			# ready to perform the requested operation
64			debugPrint(' -> removing with: ' + cmd)
65	paus	1.2	status = 0
66			if exe == 1:
67			status = os.system(cmd)
68	paus	1.4	if status != 0:
69			print ' ERROR: remove returned error %d (on %s)'%(status,cmd)
70	paus	1.1
71			return status
72
73	paus	1.5	def removeCatalog(source,debug):
74			tmp = os.getpid()
75			pid = "%d"%tmp
76			# which catalog is this one in?
77			catalogDir = '/home/cmsprod/catalog/local'
78			if re.search('/castor/cern.ch/',source):
79			catalogDir = '/home/cmsprod/catalog/cern'
80			elif re.search('/pnfs/cmsaf.mit.edu/',source) or \
81			re.search('/mnt/hadoop/cms/store/user/paus',source):
82			catalogDir = '/home/cmsprod/catalog/t2mit'
83			# now get the dataset and the book
84			f = source.split('/')
85			file = f[-1]
86			dataset = f[-2]
87			book = f[-4] + '/' + f[-3]
88
89			dir = catalogDir + '/' + book + '/' + dataset
90
91			# now remove the particular file from the record
92			cmd = 'cat ' + dir + '/RawFiles.?? \| sort -u \| grep -v ' + file + ' > /tmp/RawFiles.00.' + pid
93			execute(cmd,debug)
94			cmd = 'rm ' + dir + '/RawFiles.??'
95			execute(cmd,debug)
96			cmd = 'mv /tmp/RawFiles.00.' + pid + ' ' + dir + '/RawFiles.00'
97			execute(cmd,debug)
98			cmd = 'cat ' + dir + '/Files \| grep -v ' + file + ' > /tmp/Files.' + pid
99			execute(cmd,debug)
100			cmd = 'mv /tmp/Files.' + pid + ' ' + dir + '/Files'
101			execute(cmd,debug)
102
103			return
104
105	paus	1.1	#===================================================================================================
106			# Main starts here
107			#===================================================================================================
108			# Define string to explain usage of the script
109			usage = "Usage: remove <source>"
110	paus	1.2	usage += " --exe\n"
111	paus	1.5	usage += " --catalog\n"
112	paus	1.1	usage += " --debug\n"
113			usage += " --help\n"
114
115			# Define the valid options which can be specified and check out the command line
116	paus	1.5	valid = ['exe','catalog','debug','help']
117	paus	1.1	try:
118			opts, args = getopt.getopt(sys.argv[1:], "", valid)
119			except getopt.GetoptError, ex:
120			print usage
121			print str(ex)
122			sys.exit(1)
123
124			# --------------------------------------------------------------------------------------------------
125			# Get all parameters for the production
126			# --------------------------------------------------------------------------------------------------
127			# Set defaults for each option
128	paus	1.5	catalog = False
129			debug = False
130			exe = 0
131	paus	1.1
132			# Read new values from the command line
133			for opt, arg in opts:
134			#print ' OPT , ARG: ' + opt + ' ' + arg
135			if opt == '--help':
136			print usage
137			sys.exit(0)
138			elif opt == '--debug':
139	paus	1.5	debug = True
140			elif opt == '--catalog':
141			catalog = True
142	paus	1.2	elif opt == '--exe':
143	paus	1.5	exe = 1
144	paus	1.1
145			newArgv = []
146			for arg in sys.argv[1:]:
147			#print ' ARG: ' + arg
148			if arg[:2] == "--":
149			continue
150			else:
151			newArgv.append(arg)
152
153			# Define source and target
154			source = clean(newArgv[0])
155
156			# Test whether the source exists
157			if exists(source):
158			debugPrint("\n Removing: " + source)
159			remove(source)
160			elif not exists(source):
161			print ' ERROR: the source ('+source+') does not exist.'
162	paus	1.5
163			# Remove also the catalog entry
164			if catalog:
165			removeCatalog(source,debug)