#! /bin/sh
#
# This script monitors the backend and automatically restarts
# the backend if certain error conditions are detected. Those
# conditions are documented in the various comments below.
# Each of those comments contains the string "Check for", so you
# can find them quickly by searching for that pattern.
#
# If this script finds the backend has hit a problem more than 4
# times out of the last 15 checks, it assumes that your backend
# has hit a persistant error that requires manual intervention
# to correct, and it will exit after doing a final restart of the
# backend.
#
# Note that if this script is running when you do a normal "stop"
# of the backend, it will eventually restart the backend for you,
# even if that is not what you wanted. So if you deliberately want
# to stop the backend, you should stop this script first.
#
# Started by executing: /etc/init.d/mbemon start
# Halted by executing:  /etc/init.d/mbemon stop
#
# Script variables that you might choose to alter:
#
# mbelog: Location of the myth backend log file.
#         Defaults to the location used by Knoppmyth.
# stime:  Number of seconds to sleep between scans.
#         Defaults to 300 seconds (5 minutes).
# logpath:Path to the directory where this script will
#         place it's log file and various other diagnostic
#         information.
# log:    Path to logfile where this script will log various
#         status messages.
#
mbelog=/var/log/mythtv/mythbackend.log
stime=300
logdir=/var/log/mythtv/mbemon
log=$logdir/mbemon.log
hist=$logdir/history.log
#
#
#
mkdir -p $logdir
echo "mbemon started `date`" > $log
rm -f $hist
while true
do
  sleep $stime
  status="ok"
  #
  # Check for backend not running.
  #
  if [ "$status" == "ok" ]
  then
    xxx=`pgrep -l mythbackend`
    if [ "$xxx" == "" ]
    then
      status="Myth backend process is missing"
    fi
  fi
  #
  # Check for excessively large backend log
  #
  if [ "$status" == "ok" -a -e "$mbelog" ]
  then
    xxx=0
    xxx=`ls -s $mbelog | cut -d" " -f1`
    if [ "$xxx" -gt 1100 ]
    then
      status="Log file size of $xxx blocks is too large -- probable runaway"
    fi
  fi
  #
  # Check for "waiting or a thread.." in the log file
  #
  if [ "$status" == "ok" -a -e "$mbelog" ]
  then
    grep -q "waiting for a thread" $mbelog
    if [ $? -eq 0 ]
    then
      status="Log file contians 'waiting for a thread' error message"
    fi
  fi
  #
  # Check to see if backend is responding to quries on porrt 6544
  #
  if [ "$status" == "ok" ]
  then
    date > $logdir/status.txt
    lynx -dump http://localhost:6544/ >> $logdir/status.txt
    lstat=$?
    grep -q "Schedule" $logdir/status.txt
    gstat=$?
    if [ $lstat -ne 0 ] || [ $gstat -ne 0 ] ; then
      status="Unable to query port 6544 lstat=$lstat gstat=$gstat"
    fi
  fi
  #
  # Check results and restart backend if there is a problem.
  #
  histline="OK `date`"
  if [ "$status" != "ok" ]
  then
    histline="ERROR `date`"
    echo "" >> $log
    echo Problem encountered on `date` >> $log
    echo "  $status" >> $log
    echo "  Halting backend" >> $log
    /etc/init.d/mythtv-backend stop | sed -e "s/^/  /" >> $log
    if [ -e $mbelog ]
    then
      echo "  Capturing back end log file" >> $log
      [ ! -e $logdir/mythbackend.log-2 ] || \
        mv -f $logdir/mythbackend.log-2 $logdir/mythbackend.log-3
      [ ! -e $logdir/mythbackend.log-1 ] || \
        mv -f $logdir/mythbackend.log-1 $logdir/mythbackend.log-2
      [ ! -e $logdir/mythbackend.log ] || \
        mv -f $logdir/mythbackend.log $logdir/mythbackend.log-1
      mv -f $mbelog $logdir/mythbackend.log  | sed -e "s/^/  /" >> $log
      touch $logdir/mythbackend.log
    fi
    echo "  Restarting backend" >> $log
    /etc/init.d/mythtv-backend start | sed -e "s/^/  /" >> $log
    sleep 1
    echo "  Restarting gdm" >> $log
    /etc/init.d/gdm restart
  fi
  echo $histline >> $hist
  tail -15 $hist > $hist.2
  mv -f $hist.2 $hist
  cnt=`grep "^ERROR" $hist | wc -l`
  if [ $cnt -gt 4 ]
  then
    echo "  More than 4 of the last 15 checks have resulted in a restart" >> $log
    echo "  It appears that there is a persistant error that cannot be" >> $log
    echo "  recovered from. mbemon is giving up and exiting." >> $log
    exit
  fi
done

#!/bin/sh
#
# Start/stops the myth front end monitor
#
#

case "$1" in
        start)
                echo "Starting mbemon daemon"
                killall -q mbemon.sh
                start-stop-daemon -S --exec /usr/local/bin/mbemon.sh \
                  --name marcmbemon --background --nicelevel 19
                ;;
        stop)
                echo -n "Stopping mbemon daemon"
                killall -q mbemon.sh || echo -n " -- mbemon not alrady running"
                echo "."
                ;;
        restart|force-reload)
                $0 stop
                sleep 1
                $0 start
                ;;
        *)
                echo "Usage: /etc/init.d/mbemon {start|stop|restart|force-reload
}"
                exit 1
                ;;
esac

exit 0

Mar  8 20:45:09 mythhd kernel: NVRM: Xid: 16, Head 00000000 Count 003ada32
Mar  8 20:45:13 mythhd kernel: NVRM: Xid: 8, Channel 0000001e