Processing/bin/spawnDownload.sh

#!/bin/bash
#===================================================================================================
# Spawn download activities to optimize the bandwidth of the download process, one step closer to
# Phedex.
# 
#                                                                                   C.Paus, Nov 2011
#---------------------------------------------------------------------------------------------------
# Improvements to implement on a rainy Friday afternoon:
#  + connect back to an existing task
#  - optimize sleep time to allow processes to get to submit their missing file requests
#  - allow for priorities
#===================================================================================================

#---------------------------------------------------------------------------------------------------
findActivity()
{
  condor_q -global $USER -format "%d " ClusterId -format "%s " Cmd -format "%s \n" Args \
    | grep downloadFiles.sh | wc -l
}

#---------------------------------------------------------------------------------------------------
showAccounting()
{
  pid=$1; shift; iDownload=$1; shift; nDownloads=$1; shift; nJobs=$1; shift; nProcs=$1; shift
  echo " "
  echo " spawnDownload.sh  ==== Accounting Summary ==== "
  echo " "
  echo " Master pid         : $pid "
  echo " Last download tasks: $iDownload / $nDownloads "
  echo " Running processes  : $nProcs "
  echo " Running batch      : $nJobs "
  echo " "
}

#---------------------------------------------------------------------------------------------------
showAccountingShort()
{
  pid=$1; shift; iDownload=$1; shift; nDownloads=$1; shift; nJobs=$1; shift; nProcs=$1; shift
  echo " Downloads ($pid): $iDownload / $nDownloads  nProcs: $nProcs   nJobs: $nJobs "
}

#---------------------------------------------------------------------------------------------------
loopCondition()
{
  # command line option
  iDownload=$1; shift; nDownloads=$1; shift; nJobs=$1; shift;
  # return running condition
  if [ $iDownload -lt $nDownloads ] || [ $nJobs -gt 0 ]
  then
    echo 'true'
  else
    echo 'false'
  fi
}

#---------------------------------------------------------------------------------------------------
spawnCondition()
{
  # command line option
  iDownload=$1; shift; nDownloads=$1; shift; nJobs=$1; shift; nProcs=$1; shift
  # return running condition
  if [ $iDownload -lt $nDownloads ] && [ $nProcs -lt $NPROC_MAX ] && [ $nJobs -lt $NJOBS_MAX ]
  then
    echo 'true'
  else
    echo 'false'
  fi
}

#---------------------------------------------------------------------------------------------------
addProcess()
{
  pidFile=$1; shift; logFile=$1; shift;
  commandLine=$*
  $commandLine >& $logFile &
  pid=$!
  # put some useful information into the pid file
  file=`echo $commandLine | cut -d' ' -f2`
  download=`cat $file`
  echo $pid :: $commandLine :: $download >> $pidFile
  echo " Spawning pid: $pid ::  $commandLine :: $download"
}

#---------------------------------------------------------------------------------------------------
cleanUpPids()
{
  # read parameter
  pidFile=$1; shift
  # prepare empty temporary pid file
  rm -f $pidFile.tmp
  touch $pidFile.tmp
  echo " Cleaning Pids..."
  
  # parse the exisiting pid file

  while read line
  do
  
    pid=`echo $line | cut -d ' ' -f1`

    kill -0 $pid >& /dev/null
    if [ "$?" == "0" ]
    then
      grep ^$pid $pidFile >> $pidFile.tmp
    else
      grep ^$pid $pidFile >> $pidFile.done
    fi
  
  done < $pidFile

  # overwrite the existing with the active pids
  mv $pidFile.tmp $pidFile
}

#===================================================================================================
# Main activity starts here
#===================================================================================================
# defaults
LOGDIR=/home/cmsprod/public_html/download
# maximum number of processes and jobs
if [ "$NPROC_MAX" == "" ] 
then
  NPROC_MAX=8
else
  echo " Using environment NPROC_MAX: $NPROC_MAX"
fi
if [ "$NJOBS_MAX" == "" ] 
then
  NJOBS_MAX=80
else
  echo " Using environment NJOBS_MAX: $NJOBS_MAX"
fi
if [ "$SLEEP" == "" ] 
then
  SLEEP=30
else
  echo " Using environment SLEEP: $SLEEP"
fi

echo " "
echo " spawnDownload.sh  ==== Configuration ==== "
echo " "
echo " Maximal Jobs       : $NJOBS_MAX "
echo " Maximal Processes  : $NPROC_MAX "
echo " Sleeping time      : $SLEEP "
echo " "

# read command line arguments
INPUT_FILE=$1
if [ "$2" != "" ]
then
  MASTER_PID=$2
else
  MASTER_PID=$$
fi

# prepare pid file
pidFile=$LOGDIR/pids.$MASTER_PID
if [ "$2" == "" ]
then
  rm -f $pidFile
  touch $pidFile
  iDownload=0
else
  iDownload=`ls $LOGDIR/download.$MASTER_PID.* -1|cut -d'.' -f3|awk '{ if ($1 > n) n=$1; } END{print n}'`
fi

# prepare work accounting
nDownloads=`cat $INPUT_FILE | grep -v ^# | wc -l`
nJobs=`findActivity`
nProcs=`cat $pidFile | wc -l`
showAccounting $MASTER_PID $iDownload $nDownloads $nJobs $nProcs

startTime=$(date +%s)
nowTime=$(date +%s); duration=$(($nowTime - $startTime))

# the loop which finishes when activity has ceased
while [ "`loopCondition $iDownload $nDownloads $nJobs`" == "true" ]
do

  # short message we are still in the loop
  nowTime=$(date +%s); duration=$(($nowTime - $startTime))
  echo " "
  echo " Loop continues... (spawnDownload.sh: duration - $duration)"
  sleep $SLEEP

  # are we spawning a new process?
  if [ "`spawnCondition $iDownload $nDownloads $nJobs $nProcs`" == "true" ]
  then
    iDownload=$(( $iDownload + 1 ))
    #echo " Download file: $INPUT_FILE - line $iDownload ($nDownloads)"
    cat $INPUT_FILE | grep -v ^# | sed -n ${iDownload}p  > $LOGDIR/download.$MASTER_PID.$iDownload
    addProcess $pidFile $LOGDIR/download.$MASTER_PID.$iDownload.log download.sh \
                        $LOGDIR/download.$MASTER_PID.$iDownload
  fi

  # full accounting step
  cleanUpPids $pidFile 
  # get updated counts on batch jobs and number of processes
  nJobs=`findActivity`
  nProcs=`cat  $pidFile | wc -l`
  echo " ==== Last update before decision ==== "
  showAccountingShort $MASTER_PID $iDownload $nDownloads $nJobs $nProcs

  makeDownloadIndex.sh
  #cp $pidFile $LOGDIR/download.$MASTER_PID.*.log ~/public_html/download/

done

# cleanup the steering files
rm -f $pidFile $pidFile.tmp

exit 0;
Revision:	1.1
Committed:	Tue Feb 28 11:54:36 2012 UTC (13 years, 2 months ago) by paus
Content type:	application/x-sh
Branch:	MAIN
CVS Tags:	Mit_032, Mit_031, Mit_030, Mit_029c, Mit_029b, Mit_030_pre1, Mit_029a, Mit_029, Mit_029_pre1, Mit_028a, Mit_028, Mit_027a, Mit_027, Mit_026, Mit_025e, Mit_025d, HEAD
Log Message:	Last updates.
#	Content
1	#!/bin/bash
2	#===================================================================================================
3	# Spawn download activities to optimize the bandwidth of the download process, one step closer to
4	# Phedex.
5	#
6	# C.Paus, Nov 2011
7	#---------------------------------------------------------------------------------------------------
8	# Improvements to implement on a rainy Friday afternoon:
9	# + connect back to an existing task
10	# - optimize sleep time to allow processes to get to submit their missing file requests
11	# - allow for priorities
12	#===================================================================================================
13
14	#---------------------------------------------------------------------------------------------------
15	findActivity()
16	{
17	condor_q -global $USER -format "%d " ClusterId -format "%s " Cmd -format "%s \n" Args \
18	\| grep downloadFiles.sh \| wc -l
19	}
20
21	#---------------------------------------------------------------------------------------------------
22	showAccounting()
23	{
24	pid=$1; shift; iDownload=$1; shift; nDownloads=$1; shift; nJobs=$1; shift; nProcs=$1; shift
25	echo " "
26	echo " spawnDownload.sh ==== Accounting Summary ==== "
27	echo " "
28	echo " Master pid : $pid "
29	echo " Last download tasks: $iDownload / $nDownloads "
30	echo " Running processes : $nProcs "
31	echo " Running batch : $nJobs "
32	echo " "
33	}
34
35	#---------------------------------------------------------------------------------------------------
36	showAccountingShort()
37	{
38	pid=$1; shift; iDownload=$1; shift; nDownloads=$1; shift; nJobs=$1; shift; nProcs=$1; shift
39	echo " Downloads ($pid): $iDownload / $nDownloads nProcs: $nProcs nJobs: $nJobs "
40	}
41
42	#---------------------------------------------------------------------------------------------------
43	loopCondition()
44	{
45	# command line option
46	iDownload=$1; shift; nDownloads=$1; shift; nJobs=$1; shift;
47	# return running condition
48	if [ $iDownload -lt $nDownloads ] \|\| [ $nJobs -gt 0 ]
49	then
50	echo 'true'
51	else
52	echo 'false'
53	fi
54	}
55
56	#---------------------------------------------------------------------------------------------------
57	spawnCondition()
58	{
59	# command line option
60	iDownload=$1; shift; nDownloads=$1; shift; nJobs=$1; shift; nProcs=$1; shift
61	# return running condition
62	if [ $iDownload -lt $nDownloads ] && [ $nProcs -lt $NPROC_MAX ] && [ $nJobs -lt $NJOBS_MAX ]
63	then
64	echo 'true'
65	else
66	echo 'false'
67	fi
68	}
69
70	#---------------------------------------------------------------------------------------------------
71	addProcess()
72	{
73	pidFile=$1; shift; logFile=$1; shift;
74	commandLine=$*
75	$commandLine >& $logFile &
76	pid=$!
77	# put some useful information into the pid file
78	file=`echo $commandLine \| cut -d' ' -f2`
79	download=`cat $file`
80	echo $pid :: $commandLine :: $download >> $pidFile
81	echo " Spawning pid: $pid :: $commandLine :: $download"
82	}
83
84	#---------------------------------------------------------------------------------------------------
85	cleanUpPids()
86	{
87	# read parameter
88	pidFile=$1; shift
89	# prepare empty temporary pid file
90	rm -f $pidFile.tmp
91	touch $pidFile.tmp
92	echo " Cleaning Pids..."
93
94	# parse the exisiting pid file
95
96	while read line
97	do
98
99	pid=`echo $line \| cut -d ' ' -f1`
100
101	kill -0 $pid >& /dev/null
102	if [ "$?" == "0" ]
103	then
104	grep ^$pid $pidFile >> $pidFile.tmp
105	else
106	grep ^$pid $pidFile >> $pidFile.done
107	fi
108
109	done < $pidFile
110
111	# overwrite the existing with the active pids
112	mv $pidFile.tmp $pidFile
113	}
114
115	#===================================================================================================
116	# Main activity starts here
117	#===================================================================================================
118	# defaults
119	LOGDIR=/home/cmsprod/public_html/download
120	# maximum number of processes and jobs
121	if [ "$NPROC_MAX" == "" ]
122	then
123	NPROC_MAX=8
124	else
125	echo " Using environment NPROC_MAX: $NPROC_MAX"
126	fi
127	if [ "$NJOBS_MAX" == "" ]
128	then
129	NJOBS_MAX=80
130	else
131	echo " Using environment NJOBS_MAX: $NJOBS_MAX"
132	fi
133	if [ "$SLEEP" == "" ]
134	then
135	SLEEP=30
136	else
137	echo " Using environment SLEEP: $SLEEP"
138	fi
139
140	echo " "
141	echo " spawnDownload.sh ==== Configuration ==== "
142	echo " "
143	echo " Maximal Jobs : $NJOBS_MAX "
144	echo " Maximal Processes : $NPROC_MAX "
145	echo " Sleeping time : $SLEEP "
146	echo " "
147
148	# read command line arguments
149	INPUT_FILE=$1
150	if [ "$2" != "" ]
151	then
152	MASTER_PID=$2
153	else
154	MASTER_PID=$$
155	fi
156
157	# prepare pid file
158	pidFile=$LOGDIR/pids.$MASTER_PID
159	if [ "$2" == "" ]
160	then
161	rm -f $pidFile
162	touch $pidFile
163	iDownload=0
164	else
165	iDownload=`ls $LOGDIR/download.$MASTER_PID.* -1\|cut -d'.' -f3\|awk '{ if ($1 > n) n=$1; } END{print n}'`
166	fi
167
168	# prepare work accounting
169	nDownloads=`cat $INPUT_FILE \| grep -v ^# \| wc -l`
170	nJobs=`findActivity`
171	nProcs=`cat $pidFile \| wc -l`
172	showAccounting $MASTER_PID $iDownload $nDownloads $nJobs $nProcs
173
174	startTime=$(date +%s)
175	nowTime=$(date +%s); duration=$(($nowTime - $startTime))
176
177	# the loop which finishes when activity has ceased
178	while [ "`loopCondition $iDownload $nDownloads $nJobs`" == "true" ]
179	do
180
181	# short message we are still in the loop
182	nowTime=$(date +%s); duration=$(($nowTime - $startTime))
183	echo " "
184	echo " Loop continues... (spawnDownload.sh: duration - $duration)"
185	sleep $SLEEP
186
187	# are we spawning a new process?
188	if [ "`spawnCondition $iDownload $nDownloads $nJobs $nProcs`" == "true" ]
189	then
190	iDownload=$(( $iDownload + 1 ))
191	#echo " Download file: $INPUT_FILE - line $iDownload ($nDownloads)"
192	cat $INPUT_FILE \| grep -v ^# \| sed -n ${iDownload}p > $LOGDIR/download.$MASTER_PID.$iDownload
193	addProcess $pidFile $LOGDIR/download.$MASTER_PID.$iDownload.log download.sh \
194	$LOGDIR/download.$MASTER_PID.$iDownload
195	fi
196
197	# full accounting step
198	cleanUpPids $pidFile
199	# get updated counts on batch jobs and number of processes
200	nJobs=`findActivity`
201	nProcs=`cat $pidFile \| wc -l`
202	echo " ==== Last update before decision ==== "
203	showAccountingShort $MASTER_PID $iDownload $nDownloads $nJobs $nProcs
204
205	makeDownloadIndex.sh
206	#cp $pidFile $LOGDIR/download.$MASTER_PID.*.log ~/public_html/download/
207
208	done
209
210	# cleanup the steering files
211	rm -f $pidFile $pidFile.tmp
212
213	exit 0;