README_PRODUCTION_INSTALL

Last Revised: April 2009

This contains notes on how the MushroomObserver website was setup on the
slicehost servers and should help anyone wanting to move the site to another
server at some point in the future. 

The target slicehost configuration is a clean Fedora Core 10 512Mb slice.

Estimated time: An hour or two

################################################################################

# Basic installation.
# When running yum respond 'y' to all questions.
root> yum update
root> yum install tcsh mlocate gcc make vim lynx telnet emacs bind-utils
root> yum install ImageMagick subversion httpd mysql-devel mysql-server monit
root> yum install ruby ruby-devel rubygem-mongrel libjpeg-devel

# Setup clock to PDT (actually did this at the end so it might have to happen later)
root> cp /etc/localtime /etc/localtime-orig
root> cp /usr/share/zoneinfo/America/Los_Angeles /etc/localtime

# Check that this installs the currently supported version of rails 2.1.1.
# This can be done with: gem list
# If not see below...
root> yum install rubygem-rails

root> gem install rake mongrel_cluster RedCloth mysql --no-ri --no-rdoc
# This is expected fail.  See below...

# We need to be sure to get the right version of rails.
# Only run this if yum didn't get the right version.
root> gem install rails --version 2.1.1 --no-ri --no-rdoc

# When mysql fails to build...
root> cd /usr/lib/ruby/gems/1.8/gems/mysql*
root> ruby extconf.rb --with-mysql-config=/usr/bin/mysql_config
root> make install

# Add user "mo" which will run MO production server.
root> useradd mo

# Don't allow people to login directly as mo.
root> vi /etc/shadow
    (replace mo:!!:... with mo:*:...)

# Allow members of group "mo" ability to "sudo su mo".
root> visudo
    (add "%mo ALL = NOPASSWD: /bin/su mo" to end)

# Add users.
root> useradd -G mo -s /bin/tcsh xxx
root> useradd -G mo -s /bin/tcsh yyy
root> passwd xxx
root> passwd yyy

# Allow tcp through firewall.
root> iptables -A INPUT -p tcp --dport 80 -j ACCEPT
root> system-config-firewall-tui
  (enable WWW (HTTP))

# Test apache (ignore the warning message for now):
root> service httpd start
    (point your external browser at http://174.143.24.209/)

# Setup apache properly.
root> vi /etc/httpd/conf/httpd.conf
    (add the content given below)
root> chkconfig --level 345 httpd on
    (restart it later after mongrel and rails configured)

# Tell mysql to start up on boot.
root> chkconfig --level 345 mysqld on
root> service mysqld start

# Install MO.
root> mkdir -p /var/web/mo
root> chown mo /var/web/mo
root> chgrp mo /var/web/mo
root> su mo
mo> cd /var/web/mo
mo> svn checkout http://svn.collectivesource.com/mushroom_sightings/trunk /var/web/mo

# Get Database.
mo> mkdir sql
mo> cd sql
mo> scp xxx:mo/db/checkpoint dump.sql

# Setup mongrel cluster.
mo> exit
root> cd /var/web/mo
root> mongrel_rails cluster::configure -e production -p 3000 -a 127.0.0.1 -N 2 -c /var/web/mo
root> chmod +x /usr/lib/ruby/gems/1.8/gems/mongrel_cluster-1.0.5/resources/mongrel_cluster
root> ln -s /usr/lib/ruby/gems/1.8/gems/mongrel_cluster-1.0.5/resources/mongrel_cluster /etc/init.d/mongrel_cluster
root> mkdir /etc/mongrel_cluster
root> ln -s /var/web/mo/config/mongrel_cluster.yml /etc/mongrel_cluster/mo.yml
root> chkconfig --level 345 mongrel_cluster on
root> vi /var/web/mo/config/mongrel_cluster.yml
    (see below)

# Tweak environment.
root> su mo
mo> vi config/environments/development.rb
    (remove ruby-debug include at bottom)
mo> vi config/environments/production.rb
    (enable IMAGE_TRANSFER -- tells production server to copy uploaded images
    over to the image server via ssh)

# Setup password-less ssh to image server.
mo> ssh-keygen -t rsa
  (just press return three times, tell it to overwrite id_rsa.pub if it asks)
mo> ssh-copy-id -i ~/.ssh/id_rsa.pub xxx@images.mushroomobserver.org
  (enter password)
mo> ssh xxx@images.mushroomobserver.org
  (just a test to make sure it doesn't ask for password, then quit)

# Get Images.
mo> cd public/images
mo> mkdir thumb 640 orig
mo> cd thumb
mo> ssh -f xxx@images \(cd images.mushroomobserver.org/thumb\; ls \*.jpg \> x\; tar -T x -cf -\) | tar -xf -
mo> cd ../640
mo> ssh -f xxx@images \(cd images.mushroomobserver.org/640\; ls \*.jpg \> x\; tar -T x -cf -\) | tar -xf -

# Create database for testing.
mo> mysql -u root
mysql> delete from mysql.user where user != 'root' or host != 'localhost';
mysql> grant all privileges on *.* to 'mo'@'localhost' identified by 'password' with grant option;
mysql> set password for 'root'@'localhost' = password('xxx');
mysql> set password for 'mo'@'localhost' = password('xxx');
mysql> flush privileges;
mysql> quit;
mo> mysql -u mo -p
mysql> create database observer_production;
mysql> create database observer_development;
mysql> create database observer_test;
mysql> quit;

# Configure MO database.
mo> cp config/database.yml{-template,}
mo> vi config/database.yml
    (set database names, usernames, passwords, sockets)

# Populate database (from snapshot).
mo> mysql -u mo -p observer_development -e 'source sql/dump.sql'
mo> mysql -u mo -p observer_production -e 'source sql/dump.sql'
mo> rake db:migrate
    (this is the first time you actually run rails, lots of potential
    bugs might show up here having to do with environment, rails version
    mismatch, etc., not necessarily having anything to do with database)

# Test server real quick.
mo> script/server
    (you can test it with "lynx localhost:3000" in another login)
    (control-C out to stop it)

# You can also run the tests now if you want.
mo> rake test |& tee test.log

# Place server into development mode.
mo> exit
root> vi /etc/httpd/conf/httpd.conf
    (change RAILS_ENV from production to development)
root> vi /var/web/mo/config/mongrel_cluster.yml
    (change environment from production to development)

# Start server and test.
root> service mongrel_cluster start
root> service httpd restart
    (point your external browser at http://174.143.24.209/)

# Install process monitor.
root> vi /etc/monit.d/mo
    (see below)
root> chkconfig --level 345 monit on
root> service monit start

# Install mailserver.
root> yum install postfix mutt cyrus-sasl-md5 cyrus-sasl-ntlm cyrus-sasl-plain
root> vi /etc/postfix/main.cf
    myhostname = mail.mushroomobserver.org
    mydomain = mushroomobserver.org
    myorigin = $mydomain
    inet_interfaces = $myhostname, localhost
    mydestination = $myhostname, localhost.$mydomain, localhost,
            mail.$mydomain, www.$mydomain, ftp.$mydomain, $mydomain
    relayhost =
    smtpd_recipient_restrictions = \
      permit_sasl_authenticated,permit_mynetworks,reject_unauth_destination
    (I just stick these at the end and comment out the defaults)
root> vi /etc/aliases
    mailer-daemon:  root
    postmaster:     root
    root:           xxx,yyy
    xxx:            xxx@xxx.com
    yyy:            yyy@yyy.com
# If DNS doesn't point here yet, need to override the IP address:
root> vi /etc/hosts
    174.143.24.209  mail.mushroomobserver.org mushroomobserver.org
root> newaliases
root> service postfix start
root> chkconfig --level 345 postfix on

# # Configure SMTP AUTH:
# root> cat > /etc/pam.d/smtp << __END__
#   #%PAM-1.0
#   auth required pam_permit.so
#   account required pam_permit.so
#   session required pam_permit.so
#   password required pam_permit.so
#   __END__
# root> vi /etc/sysconfig/saslauthd
#   (make sure MECH=pam)
# root> cat > /usr/lib64/sasl2/smtpd.conf << __END__
#   pwcheck_method: auxprop
#   auxprop_plugin: sasldb
#   mech_list: plain login cram-md5 digest-md5 ntlm
#   __END__
# root> saslpasswd2 -c mo
#   <password>
# root> chmod 640 /etc/sasldb2
# root> chown root:mail /etc/sasldb2
# root> service saslauthd restart
# root> testsaslauthd -u mo -p tst -s smtp
# root> telnet 127.0.0.1 25
#   EHLO localhost
#   AUTH LOGIN
#   ...
#   MAIL FROM: from@email.address
#   RCPT TO: to@email.address
#   DATA
#   Subject: Test
#
#   Testing...
#   .
#   QUIT
# ...
# root> iptables -A INPUT -p tcp --dport 587 -j ACCEPT
# root> chkconfig --level 345 saslauthd on
# This does not work yet...

# Configure no-reply to bounce correctly.
root> useradd no-reply
root> cp /var/web/mo/script/autoreply          /usr/bin/autoreply
root> cp /var/web/mo/config/no-reply.muttrc    /home/no-reply/.muttrc
root> cp /var/web/mo/config/no-reply.autoreply /home/no-reply/.autoreply
root> cp /var/web/mo/config/no-reply.forward   /home/no-reply/.forward
root> chown no-reply:no-reply /home/no-reply/.[maf]*
root> chmod 644 /home/no-reply/.[maf]*
root> chmod 755 /usr/bin/autoreply

# Configure email setting for rails.
root> vi /var/web/mo/config/environment.rb
    ActionMailer::Base.smtp_settings = {
        :address => "mail.mushroomobserver.org",
        :domain => "mushroomobserver.org",
        :port => 25
        # I haven't figured out security yet.
        # :authentication => :login,
        # :user_name => "<username>",
        # :password => "<password>"
    }
root> service monit restart
    (there should be a way to tell the monit daemon to refresh
    its rules without restarting it...)
root> vi /home/mo/.muttrc
    set from="no-reply@mushroomobserver.org"

# Test email.
jason> vi /home/jason/.muttrc
    set from="jason@mushroomobserver.org"
    set use_from=yes
    set envelope_from=yes
    set editor=vi
jason> echo "This is a test." | mutt -s test pellaea@gmail.com
    (note yahoo "deferred" the test email, citing some problem
    with reply-to address?, but gmail accepts it)
    (make sure /etc/nsswitch.conf has "hosts: files dns" in it
    so that it looks at /etc/hosts before looking up ip on dns)
    (note also that /etc/hosts is cached via nscd, try
    "service nscd reload")

# Build jpegresize
jason> cd /var/web/mo/script
jason> gcc jpegresize.c -ljpeg -lm -O2 -o jpegresize
root> cp jpegresize /usr/bin

# Place server into production mode.
root> vi /etc/httpd/conf/httpd.conf
    (change RAILS_ENV back)
root> vi /var/web/mo/config/mongrel_cluster.yml
    (change environment back)

# Restart server and done.
root> service mongrel_cluster restart
root> service httpd restart

################################################################################

# Specific notes on making the official transition.
1) Do not remove the subdomain on the old host before DNS propogates!
2) Don't forget to generate a new gmaps api key.
3) Set IMAGE_SERVER to use the explicit IP address until DNS propogates.
4) Nslookup is failing on the new host(??)

# Order if moving database:
1) Shut down old and new servers.
2) Dump database and copy over to new database.
3) Copy images over to new server and new images server as necessary.
4) Change NS records at registrar to point to new server.
5) Add all DNS records at new server (see below).
6) Test email, ssh to images server, ...
7) Start new server.
8) Add redirection code to old server (see below).
9) Start old server.

# Order if just moving server:
1) Start new server
2) Copy images over.
3) Change NS records at registrar to point to new server.
4) Add all DNS records at new server (see below).
5) Add redirection code to old server (see below).
6) Restart old server.
7) Copy over any last-minute images.

# DNS records:
A blah.com. IP1
A www       IP1
A mail      IP1
A images    IP2
A lists     IP3
MX blah.com. mail.blah.com. (aux=0)
NS blah.com. ns1.webhost.com.
NS blah.com. ns2.webhost.com.
NS blah.com. ns3.webhost.com.

IP1 = IP of new server
IP2 = IP of image server (208.113.185.204 on dreamhost)
IP3 = IP of list server (66.33.216.120 on dreamhost)

################################################################################

# Adding new users:
root> useradd -G mo -s /bin/tcsh xxx
root> passwd xxx

# Adding another server instance:
root> vi /etc/httpd/conf/httpd.conf
    (search for BalancerMember.*3000 and add another line)
root> vi /var/web/mo/config/mongrel_cluster.yml
    (increment the number of "servers:" near bottom)
root> vi /etc/monit.d/mo
    (duplicate the mo1 section, changing "mo1" to "moN" and "3000" to "300N-1")
root> service monit stop
root> service httpd stop
root> service mongrel_cluster restart
root> service httpd start
root> service monit start
    (may be able to simply restart httpd and monit...)

################################################################################

# /etc/httpd/conf/httpd.conf
ServerName agaricus
ServerTokens Prod

NameVirtualHost *:80
<VirtualHost *:80>
    ServerName    mushroomobserver.org
    ServerAlias   mushroomobserver.org *.mushroomobserver.org
    ServerAdmin   nathan@mushroomobserver.org
    DocumentRoot  "/var/web/mo"
    ErrorLog      logs/mo-error_log
    CustomLog     logs/mo-access_log combined
    SetEnv        RAILS_ENV      production

    <Directory /var/web/mo>
        Options FollowSymLinks
        AllowOverride limit
        Order allow,deny
        Allow from all
    </Directory>

    ProxyPass /images      !
    ProxyPass /stylesheets !
    ProxyPass /javascripts !
    ProxyPass /robots.txt  !
    ProxyPass /favicon.ico !
    ProxyPass / balancer://mo_cluster/
    ProxyPreserveHost on

    <Proxy balancer://mo_cluster>
        BalancerMember http://127.0.0.1:3000
        BalancerMember http://127.0.0.1:3001
        BalancerMember http://127.0.0.1:3002
    </Proxy>

    RewriteEngine on
    RewriteRule ^/$ /index.html [QSA]

    # Redirect *all* image requests to image server.
    # RewriteRule ^/images/(.*)$ http://images.mushroomobserver.org/$1 [QSA]

    # Redirect only originals to image server.
    RewriteRule ^/images/orig/(.*)$ http://images.mushroomobserver.org/orig/$1 [QSA]

    # Rewrite to check for Rails cached page
    RewriteRule ^([^.]+)$ $1.html [QSA]

    # Redirect all non-static requests to cluster
    RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-f
    RewriteRule ^/(.*)$ balancer://mo_cluster%{REQUEST_URI} [P,QSA,L]

    AddOutputFilterByType DEFLATE text/plain text/html text/xml text/css text/javascript application/xml application/xhtml+
    BrowserMatch ^Mozilla/4 gzip-only-text/html
    BrowserMatch ^Mozilla/4.0[678] no-gzip
    BrowserMatch bMSIE !no-gzip !gzip-only-text/html
</VirtualHost>

################################################################################

# /var/web/mo/config/mongrel_cluster.yml
---
cwd: /var/web/mo
pid_file: tmp/pids/mongrel.pid
log_file: log/mongrel.log
user: mo
group: mo
port: "3000"
environment: production
address: 127.0.0.1
servers: 3
docroot: public

################################################################################

# /etc/monit.d/mo
set mailserver localhost
set mail-format { from: monitor@mushroomobserver.org }
set alert root@mushroomobserver.org

check system mushroomobserver.org
  if memory usage > 90% then alert
  if cpu usage (user) > 90% then alert
  group mo

check process mo1
  with pidfile /var/web/mo/tmp/pids/mongrel.3000.pid
  start program = "/usr/bin/mongrel_rails cluster::start -C /etc/mongrel_cluster/mo.yml --clean --only 3000"
  stop program  = "/usr/bin/mongrel_rails cluster::stop -C /etc/mongrel_cluster/mo.yml --only 3000"
  if cpu > 80% for 5 cycles then restart
  if totalmem > 115.0 MB for 5 cycles then restart
  if 20 restarts within 20 cycles then timeout
  group mo

check process mo2
  with pidfile /var/web/mo/tmp/pids/mongrel.3001.pid
  start program = "/usr/bin/mongrel_rails cluster::start -C /etc/mongrel_cluster/mo.yml --clean --only 3001"
  stop program  = "/usr/bin/mongrel_rails cluster::stop -C /etc/mongrel_cluster/mo.yml --only 3001"
  if cpu > 80% for 5 cycles then restart
  if totalmem > 115.0 MB for 5 cycles then restart
  if 20 restarts within 20 cycles then timeout
  group mo

check process mo3
  with pidfile /var/web/mo/tmp/pids/mongrel.3002.pid
  start program = "/usr/bin/mongrel_rails cluster::start -C /etc/mongrel_cluster/mo.yml --clean --only 3002"
  stop program  = "/usr/bin/mongrel_rails cluster::stop -C /etc/mongrel_cluster/mo.yml --only 3002"
  if cpu > 80% for 5 cycles then restart
  if totalmem > 115.0 MB for 5 cycles then restart
  if 20 restarts within 20 cycles then timeout
  group mo