Processing/bin/status.sh

#!/bin/bash
#---------------------------------------------------------------------------------------------------
# Determining the status of a given production sample
#---------------------------------------------------------------------------------------------------
H=`basename $0`

CATALOG=$HOME/catalog
BOOK=$1
DATASET=$2
PRINTOPT=$3

if [ "$PRINTOPT" == "-header" ]
then
  date=`date`
  printf "\nUpdated: $date\n\n%s\n" \
    'BOOK        SAMPLE                          All     T2(Catalg)   ToDo       T3(Catalg) ToDoT3'
  printf "%s\n" \
    '---------------------------------------------------------------------------------------------'
fi

DIR1=/pnfs/cmsaf.mit.edu/t2bat/cms/store/user/paus/$BOOK
DIR2=/mnt/hadoop/cms/store/user/paus/$BOOK
T3DIR=/mnt/hadoop/cmsprod/$BOOK
if [ "`list $DIR1 $DIR2 2> /dev/null | grep $DATASET`" != "" ]
then
  #echo wc -l $BOOK/$DATASET.lfns '2> /dev/null | cut -d ' ' -f 1'
  if ! [ -e "$BOOK/$DATASET.lfns" ]
  then
    echo " Warning: complete file list not available. Creating it now."
    echo " -> findSamples.py --cmssw=mc --pattern=$DATASET --remakeLfns=1 --complete --exe"
    findSamples.py --cmssw=mc --pattern=$DATASET --remakeLfns=1 --complete --exe >& /dev/null
  fi
  nAll=`wc -l $BOOK/$DATASET.lfns 2> /dev/null | cut -d ' ' -f 1`
  # how many are done on the Tier-2
  #echo list $DIR1/$DATASET $DIR2/$DATASET '2> /dev/null | grep root | wc -l'
  nDone=`list $DIR1/$DATASET $DIR2/$DATASET 2> /dev/null | grep root | wc -l`
  # how many are done on the Tier-2
  #echo cat $CATALOG/t2mit/$BOOK/$DATASET/Files '2> /dev/null | grep -v ^# | wc -l'
  nCata=`cat $CATALOG/t2mit/$BOOK/$DATASET/Files 2> /dev/null | grep -v ^# | wc -l`
  # how many are missing on the Tier-2
  nMissing=$(( ${nAll}-${nDone} ))
  # how many are available on the Tier-3
  nDoneT3=`list $T3DIR/$DATASET 2> /dev/null | grep root | wc -l`
  # how many are done on the Tier-3
  nCataT3=`cat $CATALOG/local/$BOOK/$DATASET/Files 2> /dev/null | grep -v ^# | wc -l`
  # how many are missing on the Tier-3
  nMissingT3=0
  if [ $nDoneT3 -gt 0 ]
  then
    nMissingT3=$(( ${nDone}-${nDoneT3} ))
  fi
  # event number, sample and event sizes
  nEvents=`cat $BOOK/$DATASET.lfns 2> /dev/null | awk '{n=n+$3} END {print n}'`
  size=0
  if [ "$nEvents" != "" ] && [ "`list $DIR1/$DATASET DIR2/$DATASET 2> /dev/null`" != ""  ]
  then
    size=`list $DIR1/$DATASET $DIR2/$DATASET 2> /dev/null | grep root | awk '{s=s+$1} END {print s/1024./1024./1024.}'`
  fi
  sizePerEvent=0
  if [ "$nEvents" != "" ] && [ $nEvents -gt 0 ]
  then
    sizePerEvent=`echo $nEvents $size | awk '{print $2*1024*1024/$1}'` 
  fi

  # print the result
  #echo " ECHO -- " \
  #  $BOOK $DATASET \
  #  $nAll $nDone $nCata $nMissing $nDoneT3 $nCataT3 $nMissingT3 $nEvents $size $sizePerEvent
  printf "%-11s %-28s %6d %6d(%6d) %6d - %6d(%6d) %6d %12d ev, %8.2f GB, %5.1f kB/ev\n" \
    $BOOK $DATASET \
    $nAll $nDone $nCata $nMissing $nDoneT3 $nCataT3 $nMissingT3 $nEvents $size $sizePerEvent
fi

# if [ $nDone -gt $nCata ]
# then
#   echo "catalog.sh -ceg `basename $BOOK` $DATASET --retry"
#   catalog.sh -ceg `basename $BOOK` $DATASET --retry
# fi

exit 0
Revision:	1.6
Committed:	Tue Feb 28 11:54:36 2012 UTC (13 years, 2 months ago) by paus
Content type:	application/x-sh
Branch:	MAIN
CVS Tags:	Mit_028a, Mit_028, Mit_027a, Mit_027, Mit_026, Mit_025e, Mit_025d
Changes since 1.5:	+10 -1 lines
Log Message:	Last updates.
#	User	Rev	Content
1	paus	1.1	#!/bin/bash
2			#---------------------------------------------------------------------------------------------------
3			# Determining the status of a given production sample
4			#---------------------------------------------------------------------------------------------------
5			H=`basename $0`
6
7	paus	1.3	CATALOG=$HOME/catalog
8	paus	1.1	BOOK=$1
9			DATASET=$2
10	paus	1.2	PRINTOPT=$3
11
12			if [ "$PRINTOPT" == "-header" ]
13			then
14	paus	1.3	date=`date`
15			printf "\nUpdated: $date\n\n%s\n" \
16			'BOOK SAMPLE All T2(Catalg) ToDo T3(Catalg) ToDoT3'
17			printf "%s\n" \
18			'---------------------------------------------------------------------------------------------'
19	paus	1.2	fi
20	paus	1.1
21	paus	1.5	DIR1=/pnfs/cmsaf.mit.edu/t2bat/cms/store/user/paus/$BOOK
22			DIR2=/mnt/hadoop/cms/store/user/paus/$BOOK
23	paus	1.2	T3DIR=/mnt/hadoop/cmsprod/$BOOK
24	paus	1.6	if [ "`list $DIR1 $DIR2 2> /dev/null \| grep $DATASET`" != "" ]
25	paus	1.1	then
26	paus	1.6	#echo wc -l $BOOK/$DATASET.lfns '2> /dev/null \| cut -d ' ' -f 1'
27			if ! [ -e "$BOOK/$DATASET.lfns" ]
28			then
29			echo " Warning: complete file list not available. Creating it now."
30			echo " -> findSamples.py --cmssw=mc --pattern=$DATASET --remakeLfns=1 --complete --exe"
31			findSamples.py --cmssw=mc --pattern=$DATASET --remakeLfns=1 --complete --exe >& /dev/null
32			fi
33	paus	1.4	nAll=`wc -l $BOOK/$DATASET.lfns 2> /dev/null \| cut -d ' ' -f 1`
34	paus	1.2	# how many are done on the Tier-2
35	paus	1.6	#echo list $DIR1/$DATASET $DIR2/$DATASET '2> /dev/null \| grep root \| wc -l'
36	paus	1.5	nDone=`list $DIR1/$DATASET $DIR2/$DATASET 2> /dev/null \| grep root \| wc -l`
37	paus	1.3	# how many are done on the Tier-2
38	paus	1.6	#echo cat $CATALOG/t2mit/$BOOK/$DATASET/Files '2> /dev/null \| grep -v ^# \| wc -l'
39	paus	1.3	nCata=`cat $CATALOG/t2mit/$BOOK/$DATASET/Files 2> /dev/null \| grep -v ^# \| wc -l`
40			# how many are missing on the Tier-2
41			nMissing=$(( ${nAll}-${nDone} ))
42	paus	1.2	# how many are available on the Tier-3
43			nDoneT3=`list $T3DIR/$DATASET 2> /dev/null \| grep root \| wc -l`
44	paus	1.3	# how many are done on the Tier-3
45			nCataT3=`cat $CATALOG/local/$BOOK/$DATASET/Files 2> /dev/null \| grep -v ^# \| wc -l`
46			# how many are missing on the Tier-3
47			nMissingT3=0
48			if [ $nDoneT3 -gt 0 ]
49			then
50			nMissingT3=$(( ${nDone}-${nDoneT3} ))
51			fi
52			# event number, sample and event sizes
53	paus	1.4	nEvents=`cat $BOOK/$DATASET.lfns 2> /dev/null \| awk '{n=n+$3} END {print n}'`
54			size=0
55	paus	1.5	if [ "$nEvents" != "" ] && [ "`list $DIR1/$DATASET DIR2/$DATASET 2> /dev/null`" != "" ]
56	paus	1.4	then
57	paus	1.5	size=`list $DIR1/$DATASET $DIR2/$DATASET 2> /dev/null \| grep root \| awk '{s=s+$1} END {print s/1024./1024./1024.}'`
58	paus	1.4	fi
59			sizePerEvent=0
60			if [ "$nEvents" != "" ] && [ $nEvents -gt 0 ]
61			then
62			sizePerEvent=`echo $nEvents $size \| awk '{print $210241024/$1}'`
63			fi
64
65	paus	1.3	# print the result
66	paus	1.4	#echo " ECHO -- " \
67			# $BOOK $DATASET \
68			# $nAll $nDone $nCata $nMissing $nDoneT3 $nCataT3 $nMissingT3 $nEvents $size $sizePerEvent
69	paus	1.3	printf "%-11s %-28s %6d %6d(%6d) %6d - %6d(%6d) %6d %12d ev, %8.2f GB, %5.1f kB/ev\n" \
70			$BOOK $DATASET \
71			$nAll $nDone $nCata $nMissing $nDoneT3 $nCataT3 $nMissingT3 $nEvents $size $sizePerEvent
72			fi
73	paus	1.2
74	paus	1.3	# if [ $nDone -gt $nCata ]
75			# then
76			# echo "catalog.sh -ceg `basename $BOOK` $DATASET --retry"
77			# catalog.sh -ceg `basename $BOOK` $DATASET --retry
78			# fi
79	paus	1.1
80			exit 0