#!/bin/bash
#
# Mithun R Rajan
# June 15, 2011
#
# This test is modeled off of Scott's limbo test.
# The test checks if the tcpcl keepalive backoff timer
# triggers a keepalive when the timer goes off. The 
# timer should trigger a keepalive every 30, 60, 120 seconds....
#
# The test first sends a file over TCPCL, then it shuts down ION
# on node 3 waits for 30 seconds to allow the CL to detect the
# the closed socket.  Then we send a second file while 
# the connection is closed. We then restart ION on node 3 and 
# wait for 30 seconds. When the backoff timer(30 seconds) expires it 
# will re-establish connection and resend the second file. A third
# file is sent from node 2 to node 3 to test restored connectivity.

./cleanup
sleep 1
echo "Starting ION..."
export ION_NODE_LIST_DIR=$PWD
rm -f ./ion_nodes
RETVAL=0

# Start nodes.
cd 3.ipn.tcp
./ionstart
cd ../2.ipn.tcp
./ionstart
echo "Sleeping 30 sec to give keepalive threads time to detect open sockets..."
sleep 30

# Start file receiver on node 3.
cd ../3.ipn.tcp
echo "Starting bprecvfile..."
bprecvfile ipn:3.1 &
sleep 1

# Send one file from node 2.
cd ../2.ipn.tcp
echo "Sending first file from node 2 to node 3..."
bpsendfile ipn:2.1 ipn:3.1 testfilex
sleep 2

# Verify that this file arrived.
cd ../3.ipn.tcp
COUNT=`ls -l testfile1 | egrep 915 | wc -l`
if [ $COUNT -eq 1 ]
then
	echo ""
	echo "Okay: got a copy of file x."
else
	echo ""
	echo "Error: didn't get a copy of file x."
	RETVAL=1
fi

echo ""
# Now break TCP connectivity to node 3.
echo "Breaking TCP connection to node 3..."
./ionstop
echo "Node 3 is stopped."
echo "Sleeping 30 sec to give keepalive thread time to detect closed socket..."
sleep 30

# Send second file from node 2.
cd ../2.ipn.tcp
echo "Sending second file from node 2 to node 3 (should go into limbo -- j)..."
bpsendfile ipn:2.1 ipn:3.1 testfiley
sleep 2

# Verify that this file has NOT arrived.
cd ../3.ipn.tcp
COUNT=`ls -l testfile2 | egrep 1070 | wc -l`
if [ $COUNT -eq 1 ]
then
	echo ""
	echo "Error: got a copy of file y already."
	RETVAL=1
else
	echo ""
	echo "Okay: didn't get a copy of file y."
fi

echo ""
# Restore node 3's TCP socket.
echo "Restoring TCP connectivity to node 3 to test automatic release."
cd ../3.ipn.tcp
./ionstart
sleep 1
rm testfile*
echo "Restarting bprecvfile..."
bprecvfile ipn:3.1 &
echo "Sleeping 30 sec to give keepalive thread time to detect open socket..."
sleep 30
echo ""
echo "Bundle for 2nd file should have been released from limbo -- k."

# Verify that the fourth file arrived.
COUNT=`ls -l testfile1 | egrep 1070 | wc -l`
if [ $COUNT -eq 1 ]
then
	echo ""
	echo "Okay: got copy of file y."
else
	echo ""
	echo "Error: didn't get a copy of file y."
	RETVAL=1
fi

echo ""
# Finally, send one more file from node 2 to test the new connection.
cd ../2.ipn.tcp
echo "Sending third file from node 2 to node 3 to test restored connectivity..."
bpsendfile ipn:2.1 ipn:3.1 testfilez
sleep 2

# Verify that this file arrived.
cd ../3.ipn.tcp
COUNT=`ls -l testfile2 | egrep 885 | wc -l`
if [ $COUNT -eq 1 ]
then
	echo ""
	echo "Okay: got a copy of file z."
else
	echo ""
	echo "Error: didn't get a copy of file z."
	RETVAL=1
fi

echo ""
# Shut down ION processes.
echo "Stopping ION..."
cd ../2.ipn.tcp
./ionstop &
cd ../3.ipn.tcp
./ionstop &

# Give all three nodes time to shut down, then clean up.
sleep 5
killm
echo "TCPCL Keepalive test completed."
exit $RETVAL
