#!/bin/bash
#DongJinLee, UoA - Aug-2010

hostName=$1	# ss (hostname)
numHost=$2	# e.g., 2 (hosts)
numOSD=$3	# e.g., 3 (disks, osds)
#let numHost=$numHost-1

letter=(b c d e f g h i j k l m n o p q r s t u v w x y z) #make sure to that the system starts with /dev/sd[b..]
function deleteOSD() {
	for (( j=1; j<=${numOSD}; j++ ))
	do
		id="${i}${j}"
		echo "${hostName}${i}: formating! /dev/sd[b-z]... ${id}/"
#		ssh ${hostName}${i} rm -rf /data/osd.${id}/* &  #running in parallel parallel			
		ssh ${hostName}${i} mkfs.ext4 -F /dev/sd${letter[$j-1]}  &  #running in parallel (takes a while time...)	
		sleep 1
	done
	wait
}

function mountOSD() {
	for (( j=1; j<=${numOSD}; j++ ))
	do
		id="${i}${j}"
		echo "${hostName}${i}: mounting /dev/sd${letter[$j-1]} /data/osd.${id}"
		ssh ${hostName}${i} mkdir -p /data/osd.${id}
		ssh ${hostName}${i} mount -o user_xattr /dev/sd${letter[$j-1]} /data/osd.${id}
		ssh ${hostName}${i} rm -rf /data/osd.${id}/*
	done	
	ssh ${hostName}${i} rm -rf /cephlog  #OSD
	ssh ${hostName}${i} mkdir -p /cephlog  #OSD
	ssh ${hostName}${i} rm -rf /tmp/mkcephfs.* > /dev/null 2>&1
	ssh ${hostName}${i} rm -rf /tmp/keyring-* > /dev/null 2>&1
	ssh ${hostName}${i} rm -rf /tmp/osdmap* > /dev/null 2>&1
	ssh ${hostName}${i} rm -rf /tmp/monmap* > /dev/null 2>&1
	ssh ${hostName}${i} rm -rf /tmp/monkeyring* > /dev/null 2>&1
	ssh ${hostName}${i} rm -rf /tmp/keyring* > /dev/null 2>&1	
	
}
echo ">>> 1. Stopping Ceph..."
ssh ss3 /root/ceph/ceph/src/stop.sh all > /dev/null 2>&1

echo ">>> 2. Unmounting and Removing..."
umount /media/ceph > /dev/null 2>&1

ssh ss3 rm -rf /data/mon0/
ssh ss3 mkdir -p /data/mon0/
ssh ss3 rm -rf /cephlog
ssh ss3 mkdir -p /cephlog
ssh ss3 rm -rf /tmp/mkcephfs.* > /dev/null 2>&1
ssh ss3 rm -rf /tmp/keyring-* > /dev/null 2>&1
ssh ss3 rm -rf /tmp/osdmap* > /dev/null 2>&1
ssh ss3 rm -rf /tmp/monmap* > /dev/null 2>&1
ssh ss3 rm -rf /tmp/monkeyring* > /dev/null 2>&1
ssh ss3 rm -rf /tmp/keyring* > /dev/null 2>&1	

echo "stopping OSDs"
for (( i=1; i<=${numHost}; i++ ))
do
	ssh ${hostName}${i} /root/ceph/ceph/src/stop.sh all > /dev/null 2>&1
done

echo ">>> 3. Deleting OSDs..."
for (( i=1; i<=${numHost}; i++ ))
do
	ssh ${hostName}${i} umount /data/* #> /dev/null 2>&1
	deleteOSD $i & #parallel
done
wait

for (( i=1; i<=${numHost}; i++ ))
do
	mountOSD $i & #parallel
done
echo "waiting for the mount..."
wait

echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
for (( i=1; i<=${numHost}; i++ ))
do
	echo "--- ${hostName}${i} ---"
	ssh ${hostName}${i} df -h | grep data 
done