UserCode/joshmt/onlyLast.py

import os, sys, re

###############################################################

#Usage:
# python onlyLast.py <path to input directory on CASTOR>

#now updated to handle output from the latest CRAB with
#file names like this:
#                      myfilename_1_1_aBC.root

# 02/09/11 - Don
# updated to handle files on the T3
# this script now automatically checks if the input dir is pointing
# to the T3 and adapts accordingly
# e.g. python onlyLast.py "$CUSE/blah" 

###############################################################

#handle input arguments
#this includes the name of the script
##for arg in sys.argv:
##    print arg
    
#sys.argv[1] will have the first argument
tmpfile = '.onlyLast_py_'
mypid = os.getpid()
tmpfile += `mypid`


inputdir = sys.argv[1]
if inputdir[len(inputdir)-1] != '/':
    inputdir+='/'

#check if the input directory is located on the T3
isT3 = False
checkT3 = 'echo '
checkT3 += inputdir
checkT3 += ' | awk -F "/" \'{print $1}\' '
p = os.popen(checkT3)
location = p.readline()
p.close()
#print location
if "srm" in location:
        print "T3 directory specified.  Using grid commands."
        isT3 = True


outputdir = inputdir
outputdir += 'EXTRAS'

mkdircommand = "nsmkdir "
mkdircommand += outputdir

#if isT3 == 'T3':
#    mkdircommand = "srmmkdir "
#    mkdircommand += outputdir


#get a file listing into tmpfile
cmd = 'nsls -l '
cmd += inputdir
cmd += ' | awk \'// {print $5,$9;}\' > '


if isT3 == True:
    cmd = 'srmls "'
    cmd += inputdir
    cmd += '" | awk -F "/" \'{print $1,$NF}\' | awk \'NF>0\'  > '


cmd+= tmpfile
#print cmd
os.system(cmd)

#keep track of whether we made the directory or not
alreadymadedir = 0
#do not manually make the directory for the T3, lcg-cp will take care of it
if isT3 == True:
    alreadymadedir = 1


f = open(tmpfile,'r')

#to deal with the arbitrary extension to the filename that crab
#now adds, I will add a second associative array to keep track of the
#whole file name. (completefilenames)
#The only subtle part is where the file names are sorted.
#Hopefully this does not create any bugs
indexdict = {}
completefilenames = {}

stub = 'stub'

#do some accounting
nmoved = 0

for line in f:
    mypair = line.split()
#file size
    size = int(mypair[0])
#parse filename
    result = re.match('(.*)_([0-9]*)_([0-9]*)_(.*).root',mypair[1])
    if size > 0:
        if stub=='stub':
            stub = result.group(1)
        elif stub!=result.group(1):
            print "WARNING -- there is more than one group of filenames here!"
        
        if result.group(2) in indexdict:
            #so the key already exists
            indexdict[result.group(2)].append(result.group(3))
            completefilenames[result.group(2)].append(mypair[1])
        else:
            indexdict[result.group(2)] = [result.group(3)]
            completefilenames[result.group(2)] = [mypair[1]]
#        print result.group(0)
#        print result.group(1)
#        print result.group(2)
#        print result.group(3)

#print indexdict

for ii in indexdict:
    if len(indexdict[ii])==1:
        print "nothing to do for ",ii
    else:
        indexdict[ii].sort()
        #will this work? in my one test case, yes
        completefilenames[ii].sort()
        goodindex = indexdict[ii].pop()
        goodfilename = completefilenames[ii].pop()
        print ii, ": keeping index ",goodindex
        print ii, ": corresponds to file ",goodfilename
        for jj in completefilenames[ii]:
            if alreadymadedir == 0:
                print mkdircommand
                #this will give a harmless error if the dir already exists
                os.system(mkdircommand)

            s=inputdir
            s+=jj
            print "moving ", s
            cpcmd = 'rfcp '
            if isT3 == True:
                cpcmd = 'lcg-cp --verbose -b -D srmv2 "'
            cpcmd += s

            if isT3 == True:
                cpcmd += '" "'
            else: cpcmd += ' '

            cpcmd += outputdir
            if isT3 == True:
                cpcmd +='/'
                cpcmd +=jj
                cpcmd += '"'
            print cpcmd
            os.system(cpcmd)
            rmcmd = 'rfrm '
            if isT3 == True:
                rmcmd = 'srmrm '
            rmcmd += s
            print rmcmd
            os.system(rmcmd)
            nmoved = nmoved+1

f.close()
os.remove(tmpfile)

print "----------------------------"
print "I moved this many files: ",nmoved
print "----------------------------"


Revision:	1.4
Committed:	Fri Sep 2 14:03:35 2011 UTC (13 years, 8 months ago) by wteo
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	HEAD
Changes since 1.3:	+57 -2 lines
Log Message:	add T3 directory support
#	User	Rev	Content
1	joshmt	1.1	import os, sys, re
2
3	joshmt	1.2	###############################################################
4
5			#Usage:
6			# python onlyLast.py <path to input directory on CASTOR>
7
8			#now updated to handle output from the latest CRAB with
9			#file names like this:
10			# myfilename_1_1_aBC.root
11
12	wteo	1.4	# 02/09/11 - Don
13			# updated to handle files on the T3
14			# this script now automatically checks if the input dir is pointing
15			# to the T3 and adapts accordingly
16			# e.g. python onlyLast.py "$CUSE/blah"
17
18	joshmt	1.2	###############################################################
19
20	joshmt	1.1	#handle input arguments
21			#this includes the name of the script
22			##for arg in sys.argv:
23			## print arg
24
25			#sys.argv[1] will have the first argument
26	joshmt	1.3	tmpfile = '.onlyLast_py_'
27	joshmt	1.1	mypid = os.getpid()
28			tmpfile += `mypid`
29
30	wteo	1.4
31	joshmt	1.1	inputdir = sys.argv[1]
32			if inputdir[len(inputdir)-1] != '/':
33			inputdir+='/'
34
35	wteo	1.4	#check if the input directory is located on the T3
36			isT3 = False
37			checkT3 = 'echo '
38			checkT3 += inputdir
39			checkT3 += ' \| awk -F "/" \'{print $1}\' '
40			p = os.popen(checkT3)
41			location = p.readline()
42			p.close()
43			#print location
44			if "srm" in location:
45			print "T3 directory specified. Using grid commands."
46			isT3 = True
47
48
49	joshmt	1.1	outputdir = inputdir
50			outputdir += 'EXTRAS'
51
52			mkdircommand = "nsmkdir "
53			mkdircommand += outputdir
54
55	wteo	1.4	#if isT3 == 'T3':
56			# mkdircommand = "srmmkdir "
57			# mkdircommand += outputdir
58
59
60	joshmt	1.1	#get a file listing into tmpfile
61			cmd = 'nsls -l '
62			cmd += inputdir
63			cmd += ' \| awk \'// {print $5,$9;}\' > '
64	wteo	1.4
65
66			if isT3 == True:
67			cmd = 'srmls "'
68			cmd += inputdir
69			cmd += '" \| awk -F "/" \'{print $1,$NF}\' \| awk \'NF>0\' > '
70
71
72			cmd+= tmpfile
73	joshmt	1.1	#print cmd
74			os.system(cmd)
75
76	joshmt	1.2	#keep track of whether we made the directory or not
77			alreadymadedir = 0
78	wteo	1.4	#do not manually make the directory for the T3, lcg-cp will take care of it
79			if isT3 == True:
80			alreadymadedir = 1
81
82
83
84	joshmt	1.1
85			f = open(tmpfile,'r')
86
87	joshmt	1.2	#to deal with the arbitrary extension to the filename that crab
88			#now adds, I will add a second associative array to keep track of the
89			#whole file name. (completefilenames)
90			#The only subtle part is where the file names are sorted.
91			#Hopefully this does not create any bugs
92	joshmt	1.1	indexdict = {}
93	joshmt	1.2	completefilenames = {}
94	joshmt	1.1
95			stub = 'stub'
96
97	joshmt	1.2	#do some accounting
98			nmoved = 0
99
100	joshmt	1.1	for line in f:
101			mypair = line.split()
102			#file size
103			size = int(mypair[0])
104			#parse filename
105	joshmt	1.2	result = re.match('(.)_([0-9])_([0-9])_(.).root',mypair[1])
106	joshmt	1.1	if size > 0:
107			if stub=='stub':
108			stub = result.group(1)
109			elif stub!=result.group(1):
110	joshmt	1.2	print "WARNING -- there is more than one group of filenames here!"
111	joshmt	1.1
112			if result.group(2) in indexdict:
113			#so the key already exists
114			indexdict[result.group(2)].append(result.group(3))
115	joshmt	1.2	completefilenames[result.group(2)].append(mypair[1])
116	joshmt	1.1	else:
117			indexdict[result.group(2)] = [result.group(3)]
118	joshmt	1.2	completefilenames[result.group(2)] = [mypair[1]]
119	joshmt	1.1	# print result.group(0)
120			# print result.group(1)
121			# print result.group(2)
122			# print result.group(3)
123
124			#print indexdict
125
126			for ii in indexdict:
127			if len(indexdict[ii])==1:
128			print "nothing to do for ",ii
129			else:
130			indexdict[ii].sort()
131	joshmt	1.2	#will this work? in my one test case, yes
132			completefilenames[ii].sort()
133	joshmt	1.1	goodindex = indexdict[ii].pop()
134	joshmt	1.2	goodfilename = completefilenames[ii].pop()
135	joshmt	1.1	print ii, ": keeping index ",goodindex
136	joshmt	1.2	print ii, ": corresponds to file ",goodfilename
137			for jj in completefilenames[ii]:
138			if alreadymadedir == 0:
139			print mkdircommand
140			#this will give a harmless error if the dir already exists
141			os.system(mkdircommand)
142
143	joshmt	1.1	s=inputdir
144			s+=jj
145	joshmt	1.2	print "moving ", s
146	joshmt	1.1	cpcmd = 'rfcp '
147	wteo	1.4	if isT3 == True:
148			cpcmd = 'lcg-cp --verbose -b -D srmv2 "'
149	joshmt	1.1	cpcmd += s
150	wteo	1.4
151			if isT3 == True:
152			cpcmd += '" "'
153			else: cpcmd += ' '
154
155	joshmt	1.1	cpcmd += outputdir
156	wteo	1.4	if isT3 == True:
157			cpcmd +='/'
158			cpcmd +=jj
159			cpcmd += '"'
160	joshmt	1.1	print cpcmd
161			os.system(cpcmd)
162			rmcmd = 'rfrm '
163	wteo	1.4	if isT3 == True:
164			rmcmd = 'srmrm '
165	joshmt	1.1	rmcmd += s
166			print rmcmd
167			os.system(rmcmd)
168	joshmt	1.2	nmoved = nmoved+1
169	joshmt	1.1
170			f.close()
171			os.remove(tmpfile)
172
173	joshmt	1.2	print "----------------------------"
174			print "I moved this many files: ",nmoved
175			print "----------------------------"
176	wteo	1.4
177
178