Mongrel Watcher (Rails)

Here's what I've set up in order to try ensure that Mongrel is running. (This setup tries to deal with the dreaded 503 Service Temporarily Unavailable error). The watcher should monitor the Mongrel, and try to make sure the site is running.

Every few seconds it checks whether Mongrel is running.

If there would be a problem that would cause Mongrel to crash each time shortly after it has been started, then a monitoring tool might (try to) start Mongrel many hundred times per hour.

So if my watcher finds no running Mongrel, it will try to start one, but then it will send a notification email and sleep for an hour.

The setup works for me, with one Mongrel running one site in one account.

Here are the files:

script/at_reboot:

# In the crontab I have (from crontab -l):
# @reboot cd skatestuff_freebox/current && script/at_reboot

export PATH=/usr/local/bin:/usr/bin:/bin

mongrel_port=4076
sleep 10
rm -f tmp/pids/dispatch.${mongrel_port}.pid
script/spin $mongrel_port
sleep 5
script/start_watcher $mongrel_port
script/request_loop

script/spin:

mongrel_port=$1
command="pgrep -u skatest -f 'mongrel_rails start.+-p $mongrel_port'"
if [ -z "$(echo $command | sh -s)" ]; then
  nohup ruby script/process/spawner mongrel -e production -p $mongrel_port -i 1
fi

script/start_watcher:

mongrel_port=$1
command="pgrep -u skatest -f 'script/watcher.+$mongrel_port'"
if [ -z "$(echo $command | sh -s)" ]; then
  nohup script/watcher $mongrel_port &
fi

script/watcher:

mongrel_port=$1

function get_mongrel_line {
  command="pgrep -u skatest -l -f 'mongrel_rails start.+-p $mongrel_port'"
  echo "$(echo $command | sh -s)"
}

function react_to_no_mongrel_running {
  alert="It seems there's no Mongrel running.\n"
  rm -f tmp/pids/dispatch.${mongrel_port}.pid
  script/spin $mongrel_port
  alert="$alert\nI tried to start one.\n"
  sleep 5
  mongrel_line=$(get_mongrel_line)
  if [ -z "$mongrel_line" ]; then
    alert="$alert\nIt seems there still is no Mongrel running.\n"
  else
    alert="$alert\nIt seems that now there is a Mongrel running:"
    alert="$alert\n$mongrel_line\n"
  fi
  alert="$alert\nPerhaps the app needs to be fixed?\n"
  alert="$alert\nI'm gonna sleep for one hour.\n"
  alert="$alert\nHere are the last lines of the production and mongrel logs:\n"
  alert="$alert\n$(tail -n 150 log/production.log)\n\n"
  alert="$alert\n$(tail -n 150 log/mongrel.log)\n"
  echo -e "$alert" | \
    mail -s "Watcher alert from skatestuff-freebox.org" \
    "$(tee < config/email_address)"
  sleep 1h
}

while [ 1 = 1 ]; do
  mongrel_line=$(get_mongrel_line)
  if [ -z "$mongrel_line" ]; then
    # It seems there's no Mongrel running, so
    react_to_no_mongrel_running
  fi
  sleep 2
done

Capfile:

load 'deploy' if respond_to?(:namespace) # cap2 differentiator
Dir['vendor/plugins/*/recipes/*.rb'].each { |plugin| load(plugin) }
load 'config/deploy'

after "deploy:update_code", "deploy:fix_script_perms"
after "deploy:update_code", "deploy:set_up_database_yml"
after "deploy:update_code", "deploy:set_env"
after "deploy:update_code", "deploy:link_rails"

# Sometimes this will be redundant:
before "deploy:stop", "deploy:stop_watcher"
# Works around a strange missing PID file issue, IIRC:
before "deploy:update_code", "deploy:stop_watcher"

# before "deploy:update_code", "deploy:stop_request_loop"
# after "deploy:stop", "deploy:mongrel_status"
before "deploy:stop", "deploy:stop_request_loop"

# after "deploy:stop_watcher", "deploy:watcher_status"
# after "deploy:start_watcher", "deploy:watcher_status"

after "deploy:start", "deploy:start_watcher"
# after "deploy:start", "deploy:mongrel_status"
after "deploy:start", "deploy:start_request_loop"

# before "deploy:restart", "deploy:stop_watcher"
# after "deploy:restart", "deploy:start_watcher"
# after "deploy:restart", "deploy:mongrel_status"

set :migrate_target, :latest
# before "deploy:symlink", "deploy:migrate"

after "deploy", "deploy:cleanup"

namespace :deploy do

  task :fix_script_perms do
    run "cd #{latest_release} && chmod 755 #{chmod755}"
  end

  task :set_up_database_yml do
    run "mv #{latest_release}/config/database.yml.online " +
      "#{latest_release}/config/database.yml"
  end

  task :set_env do
    run "(echo '/RAILS_ENV/s/^# //'; echo 'wq') | " +
      "ed -s #{latest_release}/config/environment.rb"
  end

  task :link_rails do
    run "ln -s ~/rails/current #{latest_release}/vendor/rails"
  end

  task :stop do
    run "cd #{current_path} && script/process/reaper -a graceful; sleep 2"
  end

  # Just a hack
  task :restart do
    deploy.stop
    # Works OK here?
    deploy.migrate
    deploy.start
  end

  task :start do
    run "cd #{current_path} && script/spin #{mongrel_port}"
  end

  task :start_watcher do
    # sleep 5 && 
    run "cd #{current_path} && nohup script/start_watcher #{mongrel_port}"
  end

  task :stop_watcher do
    run "pkill -u #{user} -f 'script/watcher #{mongrel_port}'"
  end

  task :start_request_loop do
    run "cd #{current_path} && nohup script/start_request_loop"
  end

  task :stop_request_loop do
    run "pkill -u #{user} -f 'script/request_loop'"
  end

  task :mongrel_status do
    run "echo 'Mongrel:'; " +
    " echo \"$(pgrep -u #{user} -l -f 'mongrel_rails start'" +
    " | grep -v 'pgrep')\""
  end

  task :watcher_status do
    run "echo 'Watcher:';" +
    " echo \"$(pgrep -u #{user} -l -f 'script/watcher'" +
    " | grep -v 'pgrep')\""
  end

  task :my_cleanup do
    run "cd #{latest_release} && rm -f nohup.out log/*"
  end

end