Processing/bin/cleanupLog.py

#!/usr/bin/env python
#---------------------------------------------------------------------------------------------------
# Script to cleanup the potentially bulky production output
#
# Author: C.Paus                                                                  (February 8, 2010)
#---------------------------------------------------------------------------------------------------
import os,sys,getopt,re,string

# Make list of files to consider for cleaning
#--------------------------------------------
def makeFileList(crabId):
    print ' Make the file list for crab Id '
    allFiles = []
    cmd = 'find ./' + crabId + '/res \( -name \*.stderr -o -name \*.stdout \)'
    for line in os.popen(cmd).readlines():  # run command
        line       = line[:-1]              # strip '\n'
        #print ' LINE: ' + line
        file       = line                   # splitting every blank
        #file       = f.pop()
        
        allFiles.append(file)
        #crabTask.show()
    return allFiles

# Cleanup one given file
#-----------------------
def cleanupFile(file):
    print ' Cleaning file: ' + file 

    cmd = ' gzip ' + file
    status = os.system(cmd)

    #cmd = ' mv ' + file + ' ' + file + '.original'
    #status = os.system(cmd)
    #cmd = ' cat ' + file + '.original | grep -v \'Begin processing the\' > ' + file
    #status = os.system(cmd)
    #cmd = ' rm ' + file + '.original'
    #status = os.system(cmd)
    
#===================================================================================================
# Main starts here
#===================================================================================================
# Define string to explain usage of the script
usage =  "Usage: cleanupLog.py --crabId=<name>\n"
usage += "                     --help\n"

# Define the valid options which can be specified and check out the command line
valid = ['crabId=','help']
try:
    opts, args = getopt.getopt(sys.argv[1:], "", valid)
except getopt.GetoptError, ex:
    print usage
    print str(ex)
    sys.exit(1)

# --------------------------------------------------------------------------------------------------
# Get all parameters for the production
# --------------------------------------------------------------------------------------------------
# Set defaults for each option
crabId     = None

# Read new values from the command line
for opt, arg in opts:
    if opt == "--help":
        print usage
        sys.exit(0)
    if opt == "--crabId":
        crabId = arg

# Deal with obvious problems
if crabId == None:
    cmd = "--crabId  required parameter not provided."
    raise RuntimeError, cmd

cmd = 'du -s --block-size=1 ' + crabId + ' | cut -f1'
for line in os.popen(cmd).readlines():  # run command
    sizeBefore = line[:-1]              # strip '\n'

allFiles = []
allFiles = makeFileList(crabId)

for file in allFiles:
    #print ' Clean file: %s'%(file)
    cleanupFile(file)

cmd = 'du -s --block-size=1 ' + crabId + ' | cut -f1'
for line in os.popen(cmd).readlines():  # run command
    sizeAfter = line[:-1]               # strip '\n'

sizeBeforeMb = int(sizeBefore)/1024./1024.
sizeAfterMb  = int(sizeAfter)/1024./1024.
deltaMb      = (int(sizeBefore)-int(sizeAfter))/1024./1024.

print ' '
print ' Space before:  %12.4f MB'%sizeBeforeMb
print ' Space after :  %12.4f MB'%sizeAfterMb
print ' ==================================='
print ' Gained      :  %12.4f MB'%deltaMb
print ' '
Revision:	1.2
Committed:	Sat Jun 5 02:36:28 2010 UTC (14 years, 11 months ago) by paus
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	Mit_032, Mit_031, Mit_025c_branch2, Mit_025c_branch1, Mit_030, Mit_029c, Mit_029b, Mit_030_pre1, Mit_029a, Mit_029, Mit_029_pre1, Mit_028a, Mit_025c_branch0, Mit_028, Mit_027a, Mit_027, Mit_026, Mit_025e, Mit_025d, Mit_025c, Mit_025b, Mit_025a, Mit_025, Mit_025pre2, Mit_024b, Mit_025pre1, Mit_024a, Mit_024, Mit_023, Mit_022a, Mit_022, Mit_020d, TMit_020d, Mit_020c, Mit_021, Mit_021pre2, Mit_021pre1, Mit_020b, Mit_020a, Mit_020, Mit_020pre1, Mit_018, Mit_017, Mit_017pre3, Mit_017pre2, Mit_017pre1, Mit_016, Mit_015b, Mit_015a, Mit_015, Mit_014e, Mit_014d, Mit_014c, Mit_014b, Mit_014a, HEAD
Branch point for:	Mit_025c_branch
Changes since 1.1:	+99 -0 lines
Log Message:	Wow I forgot all about my cvs.
#	User	Rev	Content
1	paus	1.2	#!/usr/bin/env python
2			#---------------------------------------------------------------------------------------------------
3			# Script to cleanup the potentially bulky production output
4			#
5			# Author: C.Paus (February 8, 2010)
6			#---------------------------------------------------------------------------------------------------
7			import os,sys,getopt,re,string
8
9			# Make list of files to consider for cleaning
10			#--------------------------------------------
11			def makeFileList(crabId):
12			print ' Make the file list for crab Id '
13			allFiles = []
14			cmd = 'find ./' + crabId + '/res \( -name \.stderr -o -name \.stdout \)'
15			for line in os.popen(cmd).readlines(): # run command
16			line = line[:-1] # strip '\n'
17			#print ' LINE: ' + line
18			file = line # splitting every blank
19			#file = f.pop()
20
21			allFiles.append(file)
22			#crabTask.show()
23			return allFiles
24
25			# Cleanup one given file
26			#-----------------------
27			def cleanupFile(file):
28			print ' Cleaning file: ' + file
29
30			cmd = ' gzip ' + file
31			status = os.system(cmd)
32
33			#cmd = ' mv ' + file + ' ' + file + '.original'
34			#status = os.system(cmd)
35			#cmd = ' cat ' + file + '.original \| grep -v \'Begin processing the\' > ' + file
36			#status = os.system(cmd)
37			#cmd = ' rm ' + file + '.original'
38			#status = os.system(cmd)
39
40			#===================================================================================================
41			# Main starts here
42			#===================================================================================================
43			# Define string to explain usage of the script
44			usage = "Usage: cleanupLog.py --crabId=<name>\n"
45			usage += " --help\n"
46
47			# Define the valid options which can be specified and check out the command line
48			valid = ['crabId=','help']
49			try:
50			opts, args = getopt.getopt(sys.argv[1:], "", valid)
51			except getopt.GetoptError, ex:
52			print usage
53			print str(ex)
54			sys.exit(1)
55
56			# --------------------------------------------------------------------------------------------------
57			# Get all parameters for the production
58			# --------------------------------------------------------------------------------------------------
59			# Set defaults for each option
60			crabId = None
61
62			# Read new values from the command line
63			for opt, arg in opts:
64			if opt == "--help":
65			print usage
66			sys.exit(0)
67			if opt == "--crabId":
68			crabId = arg
69
70			# Deal with obvious problems
71			if crabId == None:
72			cmd = "--crabId required parameter not provided."
73			raise RuntimeError, cmd
74
75			cmd = 'du -s --block-size=1 ' + crabId + ' \| cut -f1'
76			for line in os.popen(cmd).readlines(): # run command
77			sizeBefore = line[:-1] # strip '\n'
78
79			allFiles = []
80			allFiles = makeFileList(crabId)
81
82			for file in allFiles:
83			#print ' Clean file: %s'%(file)
84			cleanupFile(file)
85
86			cmd = 'du -s --block-size=1 ' + crabId + ' \| cut -f1'
87			for line in os.popen(cmd).readlines(): # run command
88			sizeAfter = line[:-1] # strip '\n'
89
90			sizeBeforeMb = int(sizeBefore)/1024./1024.
91			sizeAfterMb = int(sizeAfter)/1024./1024.
92			deltaMb = (int(sizeBefore)-int(sizeAfter))/1024./1024.
93
94			print ' '
95			print ' Space before: %12.4f MB'%sizeBeforeMb
96			print ' Space after : %12.4f MB'%sizeAfterMb
97			print ' ==================================='
98			print ' Gained : %12.4f MB'%deltaMb
99			print ' '