due to ram problems at our vserver some of our tasks stopped working in the middle of the night or while we were not around. so i wrote a script checking for the tasks in top, the socket file and so on.
#!/bin/bash
############
## CONFIG ##
############
path="/home/jonny007/scripts/" #path where the temp files and the log shall be created
logLevel=2 #level, which messages shall be written into the log. 0=off; 1=error; 2=warn; 3=info
logFile="logCheck.log" #logFile
logCutLength=15 #number of lines, which shall be cut of the log of the sevices
email="email@gmail.de"
subjectMail="check results"
contentMail="<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\"><html><head><style type=\"text/css\">body{font-family:verdana}.green{background:#040;color:#0A0}.red{background:#A00;color:#400}.left{text-align:center}.left b{text-decoration:underline;color:#000}th,table{border:none}th{font-size:110%;line-height:300%}td{border:1px solid #000}</style><title></title></head><body><table><tr><th colspan=\"2\">`date`</th></tr>"
#Normal Check
#names of all main services
serviceNameArray=( "mysql" "apache2" "postfix" "dovecot" "ssh" "postgrey")
#paths of the log files of the services
serviceLogFileArray=( "/var/log/daemon.log" "/var/log/apache2/error.log" "/var/log/mail.info" "/var/log/mail.info" "/var/log/auth.log" "/var/log/mail.info")
#command to start the services
serviceStartCommand=( "/etc/init.d/mysql start" "/etc/init.d/apache2 start" "/etc/init.d/postfix start" "/etc/init.d/dovecot start" "/etc/init.d/ssh start" "/etc/init.d/postgrey start")
#names of the sockets (only for output)
socketNameArray=( "mysqldSock" "apache2Sock" "postfixSock" "dovecotSock" "sshdSock" "postgreySock")
#path of the sockets
socketPathArray=( "/var/run/mysqld/mysqld.pid" "/var/run/apache2.pid" "/var/spool/postfix/pid/master.pid" "/var/run/dovecot/master.pid" "/var/run/sshd.pid" "/var/lib/postgrey/postgrey.lock")
#services which are children of the main services (these are the services being searched in top)
serviceSubArray=( "mysqld" "mysqld_safe" "apache2" "master" "qmgr" "dovecot" "dovecot-auth" "imap-login" "pop3-login" "sshd" "postgrey")
#to which service the subservices belong (in order of above)
serviceBelongs2Array=( "mysql" "mysql" "apache2" "postfix" "postfix" "dovecot" "dovecot" "dovecot" "dovecot" "ssh" "postgrey")
#XTended Check
#path to where all the vhosts are located
vhosts="/somethings/vhosts/"
#http hosts
hostArray=( "http://URI1" "https://URI2")
#files to query
fileArray=( "/index.html" "/robots.txt")
#subfolder from $vhosts where the file is located
sourceFolderArray=( "site1" "site2")
#address to send the test mail to
testMailAddress="noone@localmail"
#path to the mailbox of the address above
testMailboxPath="/some/thing/vmail/mailbox/new"
mysqlUser="someUser"
mysqlPwd="aPassword"
mysqlDb="database"
mysqlQuery="SELECT password FROM virtual_users WHERE user='noone'"
mysqlResult="secret"
############
## SOURCE ##
############
cd $path
logMsgTypes=("error" "warn" "info" "debug")
function funcSearch {
echo $2 | grep -q "$1" && echo 1 || echo 0
}
function funcParams {
funcHead
str="$@"
if [ "$str" = "" ]; then
funcNoParam
exit 0;
fi
if [ `funcSearch "checkconf" "$str"` = 1 -o "$1" = "cc" ]; then
funcCheckConf
fi
if [ `funcSearch "help" "$str"` = 1 ]; then
funcHelp
fi
if [ `funcSearch "all" "$str"` = 1 ]; then
nc=1
xc=1
op=1
funcCheck $1
funcCheckX $1
funcViewResults
funcStart
funcMail
exit 0;
fi
if [ `funcSearch "cron" "$str"` = 1 ]; then
nc=1
xc=1
op=1
funcCheck $1
funcCheckX $1
funcStart
funcMail
exit 0;
fi
if [ `funcSearch "normal" "$str"` = 1 -o `funcSearch "nc" "$str"` = 1 ]; then
nc=1
funcCheck $1
fi
if [ `funcSearch "xtended" "$str"` = 1 -o `funcSearch "xc" "$str"` = 1 ]; then
xc=1
funcCheckX $1
fi
if [ `funcSearch "both" "$str"` = 1 -o `funcSearch "bc" "$str"` = 1 ]; then
nc=1
xc=1
funcCheck $1
funcCheckX $1
fi
if [ `funcSearch "out" "$str"` = 1 -o `funcSearch "nco" "$str"` = 1 -o `funcSearch "xco" "$str"` = 1 -o `funcSearch "bco" "$str"` = 1 ]; then
op=1
funcViewResults
fi
if [ `funcSearch "start" "$str"` = 1 -o `funcSearch "ncs" "$str"` = 1 -o `funcSearch "ncos" "$str"` = 1 -o `funcSearch "bcos" "$str"` = 1 -o `funcSearch "bcsm" "$str"` = 1 ]; then
funcStart
fi
if [ `funcSearch "mail" "$str"` = 1 -o `funcSearch "ncm" "$str"` = 1 -o `funcSearch "ncom" "$str"` = 1 -o `funcSearch "ncosm" "$str"` = 1 -o `funcSearch "xcm" "$str"` = 1 -o `funcSearch "xcom" "$str"` = 1 -o `funcSearch "bcm" "$str"` = 1 -o `funcSearch "bcom" "$str"` = 1 -o `funcSearch "bcsm" "$str"` = 1 -o `funcSearch "bcosm" "$str"` = 1 ]; then
funcMail
fi
if [ `funcSearch "nSOCK" "$str"` = 1 ]; then
nc=1
logLevel=0
funcCheckSockets
funcViewResults
fi
if [ `funcSearch "nTOP" "$str"` = 1 ]; then
nc=1
logLevel=0
funcCheckTop
funcViewResults
fi
if [ `funcSearch "xHTTP" "$str"` = 1 ]; then
xc=1
logLevel=0
funcXCheckHTTP
funcViewResults
fi
if [ `funcSearch "xMTA" "$str"` = 1 ]; then
xc=1
logLevel=0
funcXCheckMTA
funcViewResults
fi
if [ `funcSearch "xMYSQL" "$str"` = 1 ]; then
xc=1
logLevel=0
funcXCheckMYSQL
funcViewResults
fi
}
function funcLog {
if [ $1 -le $logLevel ]; then
echo -e "`date +"%d.%m.%y %T"` ${logMsgTypes[$(($1-1))]}\t $2" >> $logFile
fi
}
function funcHead {
echo -e "\033[33mCHECK SCRIPT BY JONNY007-MKD v.0.1.1 alpha\033[1;37m\n"
}
function funcNoParam {
funcLog 3 "wrong or missing parameter: \"$1\""
echo -e "\033[0;31m\n - - - MISSING PARAMETER - - -"
echo -e " \033[0;37mTry \"help\" to show the help\n\n"
}
function funcHelp {
echo -e "\033[0;31mGive a parameter:"
echo -e "\033[0;36mhelp\t\t\033[37m: Prints this message"
echo -e "\033[0;36mcheckconf\033[0;37m || \033[0;36mcc\t\033[37m: Prints the config to show if the script is correctly configured"
echo -e "\033[0;36mcron\t\t\033[37m: Checks if the services are running, tries to start the inactives and then sends a mail"
echo -e "\033[0;36mall\t\t\033[37m: Same as cron, but also prints the results to stdout"
echo ""
echo -e "\033[0;36mnormal\t\033[37m: ONLY runs the normal top and socket check"
echo -e "\033[0;36mxtended\t\033[37m: ONLY runs the xtended check which uses the needed services"
echo -e "\033[0;36mboth\t\033[37m: Runs BOTH the normal and the xtended check"
echo -e "\033[0;36mout\t\033[37m: Prints the results"
echo -e "\033[0;36mstart\t\033[37m: Tries to start the inactive processes"
echo -e "\033[0;36mmail\t\033[37m: Sends a mail to the specified address"
echo -e "\n"
echo -e "\033[0;36mnc\t\033[37m: Same as \033[1;30mnormal"
echo -e "\033[0;36mxc\t\033[37m: Same as \033[1;30mxtended"
echo -e "\033[0;36mbc\t\033[37m: Same as \033[1;30mboth"
echo -e "\033[0;36mnco\t\033[37m: Same as \033[1;30mnormal out"
echo -e "\033[0;36mxco\t\033[37m: Same as \033[1;30mxtended out"
echo -e "\033[0;36mbco\t\033[37m: Same as \033[1;30mboth out"
echo -e "\033[0;36mncs\t\033[37m: Same as \033[1;30mnormal start"
echo -e "\033[0;36mbcs\t\033[37m: Same as \033[1;30mboth start"
echo -e "\033[0;36mncos\t\033[37m: Same as \033[1;30mnormal out start"
echo -e "\033[0;36mbcos\t\033[37m: Same as \033[1;30mboth out start"
echo -e "\033[0;36mncm\t\033[37m: Same as \033[1;30mnormal mail"
echo -e "\033[0;36mxcm\t\033[37m: Same as \033[1;30mxtended mail"
echo -e "\033[0;36mbcm\t\033[37m: Same as \033[1;30mboth mail"
echo -e "\033[0;36mncom\t\033[37m: Same as \033[1;30mnormal out mail"
echo -e "\033[0;36mxcom\t\033[37m: Same as \033[1;30mxtended out mail"
echo -e "\033[0;36mbcom\t\033[37m: Same as \033[1;30mboth out mail"
echo -e "\033[0;36mncsm\t\033[37m: Same as \033[1;30mnormal start mail"
echo -e "\033[0;36mbcsm\t\033[37m: Same as \033[1;30mboth start mail\033[0;37m or \033[1;30mcron"
echo -e "\033[0;36mncosm\t\033[37m: Same as \033[1;30mnormal out start mail"
echo -e "\033[0;36mbcosm\t\033[37m: Same as \033[1;30mboth out start mail\033[0;37m or \033[1;30mall"
echo -e "\n"
echo -e "\033[0;36mnSOCK\t\033[37m: only checks normal Sockets"
echo -e "\033[0;36mnTOP\t\033[37m: only checks normal Top"
echo -e "\033[0;36mxHTTP\t\033[37m: only checks xtended HTTP"
echo -e "\033[0;36mxMTA\t\033[37m: only checks xteneded MTA"
echo -e "\033[0;36mxMYSQL\t\033[37m: only checks xtended MySQL"
echo -e "\t\t \033[1;35mNote: When you check only one single test, the others will be shown as failed!"
echo -e "\t\t \033[1;35m Single use recommended!"
echo -e "\n"
}
function funcCheckConf {
echo -e "email: \t\t\033[1;32m$email\033[1;37m"
echo -e "subject email: \t\033[1;32m$subjectMail\033[1;37m\n"
echo -e "logFile: \t\033[1;32m$logFile\033[1;37m"
echo -ne "logLevel: \t\033[1;32m$logLevel => "
[ "$logLevel" == "0" ] && echo "nothing"
[ "$logLevel" == "1" ] && echo "error"
[ "$logLevel" == "2" ] && echo "warning"
[ "$logLevel" == "3" ] && echo "information"
[ "$logLevel" == "4" ] && echo "debug"
echo -e "\033[0;30m\n\n"
i=0
for service in ${serviceNameArray[@]}; do
echo -e " \033[41;30m$service - - - - - - - \033[40;30m":
echo -e "\t \033[1;32m${socketNameArray[i]}\033[1;37m \t is \033[1;32m${socketPathArray[i]}\033[1;37m"
echo -e "\t \033[1;34mLogfile:\033[1;37m \t is \033[1;34m${serviceLogFileArray[i]}\033[1;37m\n"
j=0
for sub in ${serviceSubArray[@]}; do
belongs2=${serviceBelongs2Array[j]}
if [ $belongs2 == $service ]; then
tmp=(${tmp[@]} $j)
fi
j=$(($j+1))
done
for bla in ${tmp[@]}; do
if [ ${#serviceSubArray[bla]} -lt 5 ]; then
tab="\t\t"
else
tab="\t"
fi
echo -e "\t \033[0;32m${serviceSubArray[bla]}\033[1;37m $tab is a child of \033[0;32m${serviceBelongs2Array[bla]}\033[40;30m"
done
unset tmp
i=$(($i+1))
done
echo -e "\n\n"
i=0
for host in ${hostArray[@]}; do
echo -e "\033[1;37mURL: \t\t\033[1;32m$host${fileArray[i]}\033[0;37m"
echo -e "\033[1;37mLocal: \t\t\033[1;32m$vhosts${sourceFolderArray[i]}${fileArray[i]}\033[0;37m"
if [ ! -f "$vhosts${sourceFolderArray[i]}${fileArray[i]}" ]; then
echo -e "\t\t\033[1;31mLocal file not found!\033[0;37m"
fi
echo
i=$(($i+1))
done
echo -e "\n\n"
echo -e "\033[1;37mtest email address: \t\033[1;32m$testMailAddress\033[0;37m"
echo -e "\033[1;37mmailbox path: \t\t\033[1;32m$testMailboxPath\033[0;37m"
echo -e "\n\n"
echo -e "\033[1;37mMySQL user: \t\t\033[1;32m$mysqlUser\033[0;37m"
echo -e "\033[1;37mMySQL passwd: \t\t\033[1;32m$mysqlPwd\033[0;37m"
echo -e "\033[1;37mMySQL database: \t\t\033[1;32m$mysqlDb\033[0;37m"
echo -e "\033[1;37mMySQL query: \t\t\033[1;32m$mysqlQuery\033[0;37m"
echo -e "\033[1;37mMySQL result: \t\t\033[1;32m$mysqlResult\033[0;37m"
}
function funcCheck {
funcLog 3 "Performing normal check with parameter \"$1\""
funcCheckSockets
funcCheckTop
i=0
sthinactive=0
while [ $i -lt ${#serviceMainChecked[@]} ]; do
if [ ${socket[i]} != 1 -o ${serviceMainChecked[i]} != 1 ]; then
funcLog 1 "Inactive found: ${serviceNameArray[i]}! Log ${serviceLogFileArray[i]} has `wc -l ${serviceLogFileArray[i]} | cut -d" " -f1` lines"
sthinactive=1
fi
i=$(($i+1))
done
unset i
}
function funcCheckX {
funcLog 3 "Performing XTended check with parameter \"$1\""
funcXCheckHTTP
funcXCheckMTA
funcXCheckMYSQL
if [ sthinactive != 1 ]; then
sthinactive=0
i=0
for result in ${resultsHTTP[@]}; do
if [ $result != 1 ]; then
sthinactive=1
break
fi
i=$(($i+1))
done
if [ $resultMTA == 0 ]; then
sthinactive=1
fi
if [ $resultMYSQL == 0 ]; then
sthinactive=1
fi
fi
}
function funcCheckSockets {
i=0;
for socketName in ${socketNameArray[@]}; do
if [ -f ${socketPathArray[i]} ]; then
funcLog 4 "n: Socket \"$socketName\" (${socketPathArray[i]}) exists"
socket[i]=1
else
funcLog 2 "n: Missing socket: $socketName in ${socketPathArray[i]}"
socket[i]=0
fi
i=$(($i+1))
done
unset i
#setting socket[] with results in order of array
}
function funcCheckTop {
top -b -n 1 > .top0.tmp
cut -d: -f2 .top0.tmp > .top1.tmp
cut -d" " -f2 -s .top1.tmp > .top2.tmp
rm .top0.tmp .top1.tmp
i=0
for service in ${serviceSubArray[@]}; do
for line in `cat .top2.tmp`; do
if [ $line == $service ]; then
funcLog 4 "n: Subservices $service found in top"
serviceChecked[i]=1
break
fi
done
i=$(($i+1))
done
rm .top2.tmp
j=0
for main in ${serviceNameArray[@]}; do
i=0
for sub in ${serviceSubArray[@]}; do
belongs2=${serviceBelongs2Array[i]}
if [ $belongs2 == $main ]; then
tmp=(${tmp[@]} $i)
fi
i=$(($i+1))
done
for bla in ${tmp[@]}; do
if [ "${serviceChecked[bla]}" = 1 ]; then
funcLog 4 "n: Subservice ${serviceSubArray[bla]} found"
blubb=1
else
funcLog 2 "n: Subservice not running: ${serviceSubArray[bla]}"
blubb=0
break
fi
done
if [ $blubb == 1 ]; then
serviceMainChecked[j]=1
else
serviceMainChecked[j]=0
fi
unset tmp bla blubb
j=$(($j+1))
done
unset i j
#setting serviceMainChecked[] with results in order of array
}
function funcXCheckHTTP {
i=0
for file in ${fileArray[@]}; do
lynx -source ${hostArray[i]}$file > .bChttp.tmp
if [ "`diff -q -N .bChttp.tmp $vhosts${sourceFolderArray[i]}$file`" = "" ]; then
funcLog 4 "x: ${hostArray[i]}! Files do not differ"
resultsHTTP[i]=1
else
resultsHTTP[i]=0
funcLog 1 "x: ${hostArray[i]}! Files differ"
fi
i=$(($i+1))
done
rm .bChttp.tmp
}
function funcXCheckMTA {
echo "test159007" | mail $testMailAddress
sleep 2
if [ "`grep -h "test159" $testMailboxPath/*`" = "test159007" ]; then
funcLog 4 "x: Mail found in mailbox $testMailboxPath"
resultMTA=1
else
resultMTA=0
funcLog 1 "x: Mail not delivered (not found in $testMailboxPath)"
fi
rm $testMailboxPath/*
}
function funcXCheckMYSQL {
mysql -u $mysqlUser -p$mysqlPwd -D $mysqlDb -s -e "$mysqlQuery" > .bCmysql.tmp
if [ "`cat .bCmysql.tmp`" = "$mysqlResult" ]; then
funcLog 4 "x: MySQL-Query succesfull"
resultMYSQL=1
else
resultMYSQL=0
funcLog 1 "x: MySQL-Query failed"
fi
rm .bCmysql.tmp
}
function funcViewResults {
if [ "$nc" = 1 ]; then
i=0
echo -e "\t\033[0;33m> NORMAL CHECK RESULTS <\033[1;37m\n"
while [ $i -lt ${#serviceNameArray[@]} ]; do
if [ "${socket[i]}" = 1 ]; then
echo -e "\033[0;32m${socketNameArray[i]} exists \033[37m(${socketPathArray[i]})"
else
echo -e "\033[0;31m${socketNameArray[i]} doesn't exist \033[37m(${socketPathArray[i]})"
fi
if [ ${serviceMainChecked[i]} == 1 ]; then
echo -e "\033[0;32m${serviceNameArray[i]} service \033[37mexists in top\n"
else
echo -e "\033[0;31m${serviceNameArray[i]} service doesn't \033[37mexist in top\n"
fi
i=$(($i+1))
done
fi
if [ "$xc" = 1 ]; then
echo -e "\t\033[0;33m> XTENDED CHECK RESULTS <\033[1;37m\n"
i=0
for result in ${resultsHTTP[@]}; do
if [ $result = 1 ]; then
echo -e "\033[0;32mHost ${hostArray[i]} successfully checked \033[37m($vhosts${sourceFolderArray[i]}${fileArray[i]})"
else
echo -e "\033[0;31mError with host ${hostArray[i]}"
fi
i=$(($i+1))
echo ""
done
if [ "$resultMTA" = 1 ]; then
echo -e "\033[0;32mMail delivered\n"
else
echo -e "\033[0;31mMail not delivered\n"
fi
if [ "$resultMYSQL" = 1 ]; then
echo -e "\033[0;32mMySQL-Query successful"
else
echo -e "\033[0;31mMySQL-Query failed"
fi
fi
}
function funcStart {
if [ "$sthinactive" = 1 ]; then
if [ "$op" = 1 ]; then
echo -e "\t\033[0;33m> START SERVICES <\033[1;37m\n"
fi
sthwasinactive=1
i=0
while [ $i -lt ${#serviceNameArray[@]} ]; do
if [ ${socket[i]} != 1 -o ${serviceMainChecked[i]} != 1 ]; then
hasBeenInAct[i]=1
else
hasBeenInAct[i]=0
fi
i=$(($i+1))
done
funcLog 2 "n: Trying to start the inactive services"
i=0
while [ $i -lt ${#serviceNameArray[@]} ]; do
if [ ${socket[i]} != 1 -o ${serviceMainChecked[i]} != 1 ]; then
funcLog 3 "n: Starting service \"${serviceNameArray[i]}\""
${serviceStartCommand[i]}
fi
i=$(($i+1))
done
sleep 5
funcLog 2 "n: Performing second (normal) check..."
funcCheck "callback"
i=0
while [ $i -lt ${#serviceNameArray[@]} ]; do
if [ ${hasBeenInAct[i]} = 1 ]; then
if [ ${serviceMainChecked[i]} = 0 ]; then
funcLog 2 "n: Service \"${serviceNameArray[i]}\" couldn't be started"
else
funcLog 2 "n: Service \"${serviceNameArray[i]}\" has been successfully started"
fi
fi
i=$(($i+1))
done
fi
}
function funcMail {
if [ "$sthinactive" = 1 -o "$sthwasinactive" = 1 ]; then
if [ "$nc" = 1 ]; then
i=0
while [ $i -lt ${#serviceNameArray[@]} ]; do
if [ ${hasBeenInAct[i]} = 1 ]; then
if [ ${serviceMainChecked[i]} = 0 ]; then
contentMail="$contentMail <tr><td class=\"left red\"><b>${serviceNameArray[i]}</b><br /> was and is still inactive! (couldnt start it)</td><td class=\"right\"><code><b>line `wc -l ${serviceLogFileArray[i]} | cut -d" " -f1` in ${serviceLogFileArray[i]}</b>"
else
contentMail="$contentMail <tr><td class=\"left green\"><b>${serviceNameArray[i]}</b><br /> was inactive but was successfully started</td><td class=\"right\"><code><b>line `wc -l ${serviceLogFileArray[i]} | cut -d" " -f1` in ${serviceLogFileArray[i]}</b>"
fi
contentMail="$contentMail `tail -n $logCutLength -q ${serviceLogFileArray[i]} > .tail.tmp; b=1; while [ $b -le $logCutLength ]; do str=\"\`sed -n ${b}p .tail.tmp\`\"; str=${str//</<\;}; echo \"<br />${str//>/>}\"; b=$(($b+1)); done`"
contentMail="$contentMail </code></td></tr>"
fi
i=$(($i+1))
done
rm .tail.tmp
fi
if [ "$xc" = 1 ]; then
i=0
for result in ${resultsHTTP[@]}; do
if [ $result != 1 ]; then
contentMail="$contentMail<tr><td class=\"left red\">Host: <b>${hostArray[i]}</b></td><td>The given files differed</td></tr><tr><td colspan=\"2\">`lynx -source ${hostArray[i]}$file > .bChttp.tmp; diff -N .bChttp.tmp $vhosts${sourceFolderArray[i]}$file`</td></tr>"
fi
i=$(($i+1))
done
if [ $resultMTA == 0 ]; then
contentMail="$contentMail<tr><td class=\"left red\">MTA</td><td>Mail was not delivered</td></tr>"
fi
if [ $resultMYSQL == 0 ]; then
contentMail="$contentMail<tr><td class=\"left red\">MYSQL</td><td>MySQL-Query didn't succeed</td></tr>"
fi
fi
contentMail="$contentMail</table></body></html>"
echo $contentMail > .contentMail.tmp
funcLog 3 "Sending mail to $email"
mutt -e "set content_type=text/html" -s "$subjectMail" "$email" < .contentMail.tmp
rm .contentMail.tmp
fi
}
funcParams $@
