[ViewVC] Annotation of: cvsroot/UserCode/MitProd/Processing/bin/remove

#!/usr/bin/env python
#---------------------------------------------------------------------------------------------------
# Script to remove a file.
#
# Author: C.Paus                                                                      (May 06, 2010)
#---------------------------------------------------------------------------------------------------
import os,sys,getopt,re,srm
t2user = os.environ['TIER2_USER']

def debugPrint(text):
    if debug:
        print ' DEBUG: ' + text

def execute(cmd,debug):
    if debug:
        print ' DEBUG: ' + cmd
    else:
        os.system(cmd)

def clean(file):
    if   re.search('dcap:',file):
        g = file.split('/')
        file = '/'.join(g[3:])
    debugPrint(' Cleaned: ' + file)

    return file


def exists(target):
    if   re.search('/castor/cern.ch/',target):
        debugPrint(' Identified a castor directory: ' + target)
        cmd = 'rfdir ' + target
    elif re.search('/pnfs/cmsaf.mit.edu/',target):
        debugPrint(' Identified a tier-2 directory: ' + target)
        cmd = 'ssh ' + t2user + '@cgate.mit.edu ls -1 ' + target + ' \>\& /dev/null'
    elif re.search('/mnt/hadoop/cms/store',target):
        debugPrint(' Identified a tier-2 hadoop directory: ' + target)
        target = srm.convertToUrl(target,debug)
        cmd = 'srmls ' + target + ' >& /dev/null'
    else:
        debugPrint(' Identified a normal directory: ' + target)
        cmd = 'ls -1 ' + target + '>& /dev/null'
            
    status = os.system(cmd)
    debugPrint(' Status: %d (on %s)' %(status,cmd))

    return (status == 0)

def remove(source):
    if   re.search('/castor/cern.ch/',source):
        debugPrint(' Identified castor file')
        cmd = "stager_rm -M " + source + "; nsrm " + source
    elif re.search('/pnfs/cmsaf.mit.edu/',source):
        debugPrint(' Identified tier-2 file')
        cmd = 'ssh ' + t2user + '@cgate.mit.edu rm -rf ' + source
    elif re.search('/mnt/hadoop/cms/store/user/paus',source):
        debugPrint(' Identified a tier-2 hadoop directory: ' + source)
        source = srm.convertToUrl(source,debug)
        cmd = 'srmrm ' + source + ' >& /dev/null'
    else:
        debugPrint(' Identified a normal directory')
        cmd = 'rm -rf ' + source

    # ready to perform the requested operation
    debugPrint('  -> removing with: ' + cmd)
    status = 0
    if exe == 1:
        status = os.system(cmd)
        if status != 0:
            print ' ERROR: remove returned error %d (on %s)'%(status,cmd)

    return status

def removeCatalog(source,debug):
    tmp = os.getpid()
    pid = "%d"%tmp
    # which catalog is this one in?
    catalogDir = '/home/cmsprod/catalog/local'
    if   re.search('/castor/cern.ch/',source):
        catalogDir = '/home/cmsprod/catalog/cern'
    elif re.search('/pnfs/cmsaf.mit.edu/',source) or \
         re.search('/mnt/hadoop/cms/store/user/paus',source):
        catalogDir = '/home/cmsprod/catalog/t2mit'
    # now get the dataset and the book
    f       = source.split('/')
    file    = f[-1]
    dataset = f[-2]
    book    = f[-4] + '/' + f[-3]

    dir     = catalogDir + '/' + book + '/' + dataset

    # now remove the particular file from the record
    cmd = 'cat ' + dir + '/RawFiles.?? | sort -u | grep -v ' + file + ' > /tmp/RawFiles.00.' + pid
    execute(cmd,debug)
    cmd = 'rm ' + dir + '/RawFiles.??'
    execute(cmd,debug)
    cmd = 'mv /tmp/RawFiles.00.' + pid + ' ' + dir + '/RawFiles.00'
    execute(cmd,debug)
    cmd = 'cat ' + dir + '/Files | grep -v ' + file + ' > /tmp/Files.' + pid
    execute(cmd,debug)
    cmd = 'mv /tmp/Files.' + pid + ' ' + dir + '/Files'
    execute(cmd,debug)

    return

#===================================================================================================
# Main starts here
#===================================================================================================
# Define string to explain usage of the script
usage =  "Usage: remove  <source>"
usage += "               --exe\n"
usage += "               --catalog\n"
usage += "               --debug\n"
usage += "               --help\n"

# Define the valid options which can be specified and check out the command line
valid = ['exe','catalog','debug','help']
try:
    opts, args = getopt.getopt(sys.argv[1:], "", valid)
except getopt.GetoptError, ex:
    print usage
    print str(ex)
    sys.exit(1)

# --------------------------------------------------------------------------------------------------
# Get all parameters for the production
# --------------------------------------------------------------------------------------------------
# Set defaults for each option
catalog = False
debug   = False
exe     = 0

# Read new values from the command line
for opt, arg in opts:
    #print ' OPT , ARG: ' + opt + ' ' + arg
    if   opt == '--help':
        print usage
        sys.exit(0)
    elif opt == '--debug':
        debug   = True
    elif opt == '--catalog':
        catalog = True
    elif opt == '--exe':
        exe     = 1

newArgv = []
for arg in sys.argv[1:]:
    #print ' ARG: ' + arg
    if arg[:2] == "--":
        continue
    else:
        newArgv.append(arg)

# Define source and target
source = clean(newArgv[0])

# Test whether the source exists
if   exists(source):
    debugPrint("\n Removing: " + source)
    remove(source)
elif not exists(source):
    print ' ERROR: the source ('+source+') does not exist.'

# Remove also the catalog entry
if catalog:
    removeCatalog(source,debug)
Revision:	1.6
Committed:	Wed Jul 25 03:05:45 2012 UTC (12 years, 9 months ago) by paus
Branch:	MAIN
CVS Tags:	Mit_029_pre1
Changes since 1.5:	+3 -2 lines
Log Message:	Preparing for new tag (Mit_029).
#	User	Rev	Content
1	paus	1.1	#!/usr/bin/env python
2			#---------------------------------------------------------------------------------------------------
3	paus	1.3	# Script to remove a file.
4	paus	1.1	#
5			# Author: C.Paus (May 06, 2010)
6			#---------------------------------------------------------------------------------------------------
7	paus	1.4	import os,sys,getopt,re,srm
8	paus	1.6	t2user = os.environ['TIER2_USER']
9	paus	1.1
10			def debugPrint(text):
11	paus	1.4	if debug:
12	paus	1.1	print ' DEBUG: ' + text
13
14	paus	1.5	def execute(cmd,debug):
15			if debug:
16			print ' DEBUG: ' + cmd
17			else:
18			os.system(cmd)
19
20	paus	1.1	def clean(file):
21			if re.search('dcap:',file):
22			g = file.split('/')
23			file = '/'.join(g[3:])
24			debugPrint(' Cleaned: ' + file)
25
26			return file
27
28
29			def exists(target):
30			if re.search('/castor/cern.ch/',target):
31			debugPrint(' Identified a castor directory: ' + target)
32			cmd = 'rfdir ' + target
33			elif re.search('/pnfs/cmsaf.mit.edu/',target):
34			debugPrint(' Identified a tier-2 directory: ' + target)
35	paus	1.6	cmd = 'ssh ' + t2user + '@cgate.mit.edu ls -1 ' + target + ' \>\& /dev/null'
36	paus	1.4	elif re.search('/mnt/hadoop/cms/store',target):
37			debugPrint(' Identified a tier-2 hadoop directory: ' + target)
38			target = srm.convertToUrl(target,debug)
39			cmd = 'srmls ' + target + ' >& /dev/null'
40	paus	1.1	else:
41			debugPrint(' Identified a normal directory: ' + target)
42			cmd = 'ls -1 ' + target + '>& /dev/null'
43
44			status = os.system(cmd)
45	paus	1.4	debugPrint(' Status: %d (on %s)' %(status,cmd))
46	paus	1.1
47			return (status == 0)
48
49			def remove(source):
50			if re.search('/castor/cern.ch/',source):
51			debugPrint(' Identified castor file')
52			cmd = "stager_rm -M " + source + "; nsrm " + source
53			elif re.search('/pnfs/cmsaf.mit.edu/',source):
54			debugPrint(' Identified tier-2 file')
55	paus	1.6	cmd = 'ssh ' + t2user + '@cgate.mit.edu rm -rf ' + source
56	paus	1.4	elif re.search('/mnt/hadoop/cms/store/user/paus',source):
57			debugPrint(' Identified a tier-2 hadoop directory: ' + source)
58			source = srm.convertToUrl(source,debug)
59			cmd = 'srmrm ' + source + ' >& /dev/null'
60	paus	1.1	else:
61			debugPrint(' Identified a normal directory')
62	paus	1.2	cmd = 'rm -rf ' + source
63	paus	1.1
64			# ready to perform the requested operation
65			debugPrint(' -> removing with: ' + cmd)
66	paus	1.2	status = 0
67			if exe == 1:
68			status = os.system(cmd)
69	paus	1.4	if status != 0:
70			print ' ERROR: remove returned error %d (on %s)'%(status,cmd)
71	paus	1.1
72			return status
73
74	paus	1.5	def removeCatalog(source,debug):
75			tmp = os.getpid()
76			pid = "%d"%tmp
77			# which catalog is this one in?
78			catalogDir = '/home/cmsprod/catalog/local'
79			if re.search('/castor/cern.ch/',source):
80			catalogDir = '/home/cmsprod/catalog/cern'
81			elif re.search('/pnfs/cmsaf.mit.edu/',source) or \
82			re.search('/mnt/hadoop/cms/store/user/paus',source):
83			catalogDir = '/home/cmsprod/catalog/t2mit'
84			# now get the dataset and the book
85			f = source.split('/')
86			file = f[-1]
87			dataset = f[-2]
88			book = f[-4] + '/' + f[-3]
89
90			dir = catalogDir + '/' + book + '/' + dataset
91
92			# now remove the particular file from the record
93			cmd = 'cat ' + dir + '/RawFiles.?? \| sort -u \| grep -v ' + file + ' > /tmp/RawFiles.00.' + pid
94			execute(cmd,debug)
95			cmd = 'rm ' + dir + '/RawFiles.??'
96			execute(cmd,debug)
97			cmd = 'mv /tmp/RawFiles.00.' + pid + ' ' + dir + '/RawFiles.00'
98			execute(cmd,debug)
99			cmd = 'cat ' + dir + '/Files \| grep -v ' + file + ' > /tmp/Files.' + pid
100			execute(cmd,debug)
101			cmd = 'mv /tmp/Files.' + pid + ' ' + dir + '/Files'
102			execute(cmd,debug)
103
104			return
105
106	paus	1.1	#===================================================================================================
107			# Main starts here
108			#===================================================================================================
109			# Define string to explain usage of the script
110			usage = "Usage: remove <source>"
111	paus	1.2	usage += " --exe\n"
112	paus	1.5	usage += " --catalog\n"
113	paus	1.1	usage += " --debug\n"
114			usage += " --help\n"
115
116			# Define the valid options which can be specified and check out the command line
117	paus	1.5	valid = ['exe','catalog','debug','help']
118	paus	1.1	try:
119			opts, args = getopt.getopt(sys.argv[1:], "", valid)
120			except getopt.GetoptError, ex:
121			print usage
122			print str(ex)
123			sys.exit(1)
124
125			# --------------------------------------------------------------------------------------------------
126			# Get all parameters for the production
127			# --------------------------------------------------------------------------------------------------
128			# Set defaults for each option
129	paus	1.5	catalog = False
130			debug = False
131			exe = 0
132	paus	1.1
133			# Read new values from the command line
134			for opt, arg in opts:
135			#print ' OPT , ARG: ' + opt + ' ' + arg
136			if opt == '--help':
137			print usage
138			sys.exit(0)
139			elif opt == '--debug':
140	paus	1.5	debug = True
141			elif opt == '--catalog':
142			catalog = True
143	paus	1.2	elif opt == '--exe':
144	paus	1.5	exe = 1
145	paus	1.1
146			newArgv = []
147			for arg in sys.argv[1:]:
148			#print ' ARG: ' + arg
149			if arg[:2] == "--":
150			continue
151			else:
152			newArgv.append(arg)
153
154			# Define source and target
155			source = clean(newArgv[0])
156
157			# Test whether the source exists
158			if exists(source):
159			debugPrint("\n Removing: " + source)
160			remove(source)
161			elif not exists(source):
162			print ' ERROR: the source ('+source+') does not exist.'
163	paus	1.5
164			# Remove also the catalog entry
165			if catalog:
166			removeCatalog(source,debug)