UserCode/joshmt/onlyLast.py

import os, sys, re

###############################################################

#Usage:
# python onlyLast.py <path to input directory on CASTOR>

#now updated to handle output from the latest CRAB with
#file names like this:
#                      myfilename_1_1_aBC.root

###############################################################

#handle input arguments
#this includes the name of the script
##for arg in sys.argv:
##    print arg
    
#sys.argv[1] will have the first argument
tmpfile = '.onlyLast_py_'
mypid = os.getpid()
tmpfile += `mypid`

inputdir = sys.argv[1]
if inputdir[len(inputdir)-1] != '/':
    inputdir+='/'

outputdir = inputdir
outputdir += 'EXTRAS'

mkdircommand = "nsmkdir "
mkdircommand += outputdir

#get a file listing into tmpfile
cmd = 'nsls -l '
cmd += inputdir
cmd += ' | awk \'// {print $5,$9;}\' > '
cmd += tmpfile
#print cmd
os.system(cmd)

#keep track of whether we made the directory or not
alreadymadedir = 0

f = open(tmpfile,'r')

#to deal with the arbitrary extension to the filename that crab
#now adds, I will add a second associative array to keep track of the
#whole file name. (completefilenames)
#The only subtle part is where the file names are sorted.
#Hopefully this does not create any bugs
indexdict = {}
completefilenames = {}

stub = 'stub'

#do some accounting
nmoved = 0

for line in f:
    mypair = line.split()
#file size
    size = int(mypair[0])
#parse filename
    result = re.match('(.*)_([0-9]*)_([0-9]*)_(.*).root',mypair[1])
    if size > 0:
        if stub=='stub':
            stub = result.group(1)
        elif stub!=result.group(1):
            print "WARNING -- there is more than one group of filenames here!"
        
        if result.group(2) in indexdict:
            #so the key already exists
            indexdict[result.group(2)].append(result.group(3))
            completefilenames[result.group(2)].append(mypair[1])
        else:
            indexdict[result.group(2)] = [result.group(3)]
            completefilenames[result.group(2)] = [mypair[1]]
#        print result.group(0)
#        print result.group(1)
#        print result.group(2)
#        print result.group(3)

#print indexdict

for ii in indexdict:
    if len(indexdict[ii])==1:
        print "nothing to do for ",ii
    else:
        indexdict[ii].sort()
        #will this work? in my one test case, yes
        completefilenames[ii].sort()
        goodindex = indexdict[ii].pop()
        goodfilename = completefilenames[ii].pop()
        print ii, ": keeping index ",goodindex
        print ii, ": corresponds to file ",goodfilename
        for jj in completefilenames[ii]:
            if alreadymadedir == 0:
                print mkdircommand
                #this will give a harmless error if the dir already exists
                os.system(mkdircommand)

            s=inputdir
            s+=jj
            print "moving ", s
            cpcmd = 'rfcp '
            cpcmd += s
            cpcmd += ' '
            cpcmd += outputdir
            print cpcmd
            os.system(cpcmd)
            rmcmd = 'rfrm '
            rmcmd += s
            print rmcmd
            os.system(rmcmd)
            nmoved = nmoved+1

f.close()
os.remove(tmpfile)

print "----------------------------"
print "I moved this many files: ",nmoved
print "----------------------------"
Revision:	1.3
Committed:	Wed Aug 25 10:24:35 2010 UTC (14 years, 8 months ago) by joshmt
Content type:	text/x-python
Branch:	MAIN
Changes since 1.2:	+1 -1 lines
Log Message:	remove hardcoding of my username
#	User	Rev	Content
1	joshmt	1.1	import os, sys, re
2
3	joshmt	1.2	###############################################################
4
5			#Usage:
6			# python onlyLast.py <path to input directory on CASTOR>
7
8			#now updated to handle output from the latest CRAB with
9			#file names like this:
10			# myfilename_1_1_aBC.root
11
12			###############################################################
13
14	joshmt	1.1	#handle input arguments
15			#this includes the name of the script
16			##for arg in sys.argv:
17			## print arg
18
19			#sys.argv[1] will have the first argument
20	joshmt	1.3	tmpfile = '.onlyLast_py_'
21	joshmt	1.1	mypid = os.getpid()
22			tmpfile += `mypid`
23
24			inputdir = sys.argv[1]
25			if inputdir[len(inputdir)-1] != '/':
26			inputdir+='/'
27
28			outputdir = inputdir
29			outputdir += 'EXTRAS'
30
31			mkdircommand = "nsmkdir "
32			mkdircommand += outputdir
33
34			#get a file listing into tmpfile
35			cmd = 'nsls -l '
36			cmd += inputdir
37			cmd += ' \| awk \'// {print $5,$9;}\' > '
38			cmd += tmpfile
39			#print cmd
40			os.system(cmd)
41
42	joshmt	1.2	#keep track of whether we made the directory or not
43			alreadymadedir = 0
44	joshmt	1.1
45			f = open(tmpfile,'r')
46
47	joshmt	1.2	#to deal with the arbitrary extension to the filename that crab
48			#now adds, I will add a second associative array to keep track of the
49			#whole file name. (completefilenames)
50			#The only subtle part is where the file names are sorted.
51			#Hopefully this does not create any bugs
52	joshmt	1.1	indexdict = {}
53	joshmt	1.2	completefilenames = {}
54	joshmt	1.1
55			stub = 'stub'
56
57	joshmt	1.2	#do some accounting
58			nmoved = 0
59
60	joshmt	1.1	for line in f:
61			mypair = line.split()
62			#file size
63			size = int(mypair[0])
64			#parse filename
65	joshmt	1.2	result = re.match('(.)_([0-9])_([0-9])_(.).root',mypair[1])
66	joshmt	1.1	if size > 0:
67			if stub=='stub':
68			stub = result.group(1)
69			elif stub!=result.group(1):
70	joshmt	1.2	print "WARNING -- there is more than one group of filenames here!"
71	joshmt	1.1
72			if result.group(2) in indexdict:
73			#so the key already exists
74			indexdict[result.group(2)].append(result.group(3))
75	joshmt	1.2	completefilenames[result.group(2)].append(mypair[1])
76	joshmt	1.1	else:
77			indexdict[result.group(2)] = [result.group(3)]
78	joshmt	1.2	completefilenames[result.group(2)] = [mypair[1]]
79	joshmt	1.1	# print result.group(0)
80			# print result.group(1)
81			# print result.group(2)
82			# print result.group(3)
83
84			#print indexdict
85
86			for ii in indexdict:
87			if len(indexdict[ii])==1:
88			print "nothing to do for ",ii
89			else:
90			indexdict[ii].sort()
91	joshmt	1.2	#will this work? in my one test case, yes
92			completefilenames[ii].sort()
93	joshmt	1.1	goodindex = indexdict[ii].pop()
94	joshmt	1.2	goodfilename = completefilenames[ii].pop()
95	joshmt	1.1	print ii, ": keeping index ",goodindex
96	joshmt	1.2	print ii, ": corresponds to file ",goodfilename
97			for jj in completefilenames[ii]:
98			if alreadymadedir == 0:
99			print mkdircommand
100			#this will give a harmless error if the dir already exists
101			os.system(mkdircommand)
102
103	joshmt	1.1	s=inputdir
104			s+=jj
105	joshmt	1.2	print "moving ", s
106	joshmt	1.1	cpcmd = 'rfcp '
107			cpcmd += s
108			cpcmd += ' '
109			cpcmd += outputdir
110			print cpcmd
111			os.system(cpcmd)
112			rmcmd = 'rfrm '
113			rmcmd += s
114			print rmcmd
115			os.system(rmcmd)
116	joshmt	1.2	nmoved = nmoved+1
117	joshmt	1.1
118			f.close()
119			os.remove(tmpfile)
120
121	joshmt	1.2	print "----------------------------"
122			print "I moved this many files: ",nmoved
123			print "----------------------------"