====== App server ======
###start
Start with adding the user:
adduser expprodl
login as expprodl
login as expprodl
ssh-keygen -t rsa
mkdir -p ~/.ssh/
cd ~/.ssh/
cat id_rsa.pub >> authorized_keys
chmod 700 ~/.ssh/
chmod 640 ~/.ssh/authorized_keys
Next, setup key based auth for devops.
#Note the following command needs an IP ADDRESS.
cat ~/.ssh/id_rsa_devops.pub | ssh expprodl@ "cat >> ~/.ssh/authorized_keys"
Grant sudo power to expprodl
//Copy the code block after the block before you exec the following command // \\
* nano /etc/sudoers
## Sudoers allows particular users to run various commands as
## the root user, without needing the root password.
##
## Examples are provided at the bottom of the file for collections
## of related commands, which can then be delegated out to particular
## users or groups.
##
## This file must be edited with the 'visudo' command.
## Host Aliases
## Groups of machines. You may prefer to use hostnames (perhap using
## wildcards for entire domains) or IP addresses instead.
# Host_Alias FILESERVERS = fs1, fs2
# Host_Alias MAILSERVERS = smtp, smtp2
## User Aliases
## These aren't often necessary, as you can use regular groups
## (ie, from files, LDAP, NIS, etc) in this file - just use %groupname
## rather than USERALIAS
# User_Alias ADMINS = jsmith, mikem
## Command Aliases
## These are groups of related commands...
## Networking
#Cmnd_Alias NETWORKING = /sbin/route, /sbin/ifconfig, /bin/ping, /sbin/dhclient, /usr/bin/net, /sbin/iptables, /usr/bin/rfcomm, /usr/bin/wvdial, /sbin/iwconfig, /sbin/mii-tool
## Installation and management of software
#Cmnd_Alias SOFTWARE = /bin/rpm, /usr/bin/up2date, /usr/bin/yum
## Services
#Cmnd_Alias SERVICES = /sbin/service, /sbin/chkconfig
## Updating the locate database
#Cmnd_Alias LOCATE = /usr/bin/updatedb
## Storage
#Cmnd_Alias STORAGE = /sbin/fdisk, /sbin/sfdisk, /sbin/parted, /sbin/partprobe, /bin/mount, /bin/umount
## Delegating permissions
#Cmnd_Alias DELEGATING = /usr/sbin/visudo, /bin/chown, /bin/chmod, /bin/chgrp
## Processes
#Cmnd_Alias PROCESSES = /bin/nice, /bin/kill, /usr/bin/kill, /usr/bin/killall
## Drivers
#Cmnd_Alias DRIVERS = /sbin/modprobe
Cmnd_Alias APACHE = /opt/apache2/bin/apachectl, /bin/vi /opt/apache2/conf/httpd.conf
Cmnd_Alias MYSQL = /sbin/service mysql stop, /sbin/service mysql start, /sbin/service mysql status, /sbin/service mysql restart
Cmnd_Alias IPTABLES = /sbin/service iptables stop, /sbin/service iptables start, /sbin/iptables
# Defaults specification
#
# Disable "ssh hostname sudo ", because it will show the password in clear.
# You have to run "ssh -t hostname sudo ".
#
#Defaults requiretty
#
# Refuse to run if unable to disable echo on the tty. This setting should also be
# changed in order to be able to use sudo without a tty. See requiretty above.
#
Defaults !visiblepw
Defaults env_reset
Defaults env_keep = "PATH COLORS DISPLAY HOSTNAME HISTSIZE INPUTRC KDEDIR \
LS_COLORS MAIL PS1 PS2 QTDIR USERNAME \
LANG LC_ADDRESS LC_CTYPE LC_COLLATE LC_IDENTIFICATION \
LC_MEASUREMENT LC_MESSAGES LC_MONETARY LC_NAME LC_NUMERIC \
LC_PAPER LC_TELEPHONE LC_TIME LC_ALL LANGUAGE LINGUAS \
_XKB_CHARSET XAUTHORITY"
## Next comes the main part: which users can run what software on
## which machines (the sudoers file can be shared between multiple
## systems).
## Syntax:
##
## user MACHINE=COMMANDS
##
## The COMMANDS section may have other options added to it.
##
## Allow root to run any commands anywhere
root ALL=(ALL) ALL
## Allows members of the 'sys' group to run networking, software,
## service management apps and more.
# %sys ALL = NETWORKING, SOFTWARE, SERVICES, STORAGE, DELEGATING, PROCESSES, LOCATE, DRIVERS
## Allows people in group wheel to run all commands
# %wheel ALL=(ALL) ALL
## Same thing without a password
# %wheel ALL=(ALL) NOPASSWD: ALL
## Allows members of the users group to mount and unmount the
## cdrom as root
# %users ALL=/sbin/mount /mnt/cdrom, /sbin/umount /mnt/cdrom
## Allows members of the users group to shutdown this system
# %users localhost=/sbin/shutdown -h now
expprodl ALL= NOPASSWD: APACHE, MYSQL, IPTABLES
We will need to install the following as we will be compiling some packages:
Create a folder to download and store the necessary software\\
mkdir -p /soft
Turn off the firewall from starting up on system restart\\
chkconfig iptables off
chkconfig ip6tables off
Turn off selinux from interfering with installs, ports\\
sed -i 's/enforcing/disabled/g' /etc/selinux/config
Update default system libraries\\
yum update
init 6 (reboot System)
We will need to install the following as we will be compiling some packages\\
yum install nano lynx wget vim at yasm yasm-devel zip unzip nasm bzip2 libyaml libyaml-devel -y
yum groupinstall "Development Tools" -y
yum groupinstall "Additional Development" -y
yum install xorg-x11-server-Xvfb -y
====== Install EPEL and additional repositories on CentOS ======
yum install epel-release -y
====== Install sendmail ======
Remove postfix \\
yum remove postfix
yum install sendmail
chkconfig sendmail on
====== Downloading Packages ======
cd /
ftp ftp.expertus.com
(After login)
cd learnexa_repo
get learnexa_packages.tgz
tar xvf learnexa_packages.tgz
====== Install Openssl ======
mv /usr/bin/openssl /usr/bin/openssl-old
mv /usr/include/openssl /usr/include/openssl-old
mv /usr/lib64/openssl /usr/lib64/openssl-old
mv /etc/pki/ca-trust/extracted/openssl /etc/pki/ca-trust/extracted/openssl-old
cd /soft && wget http://www.openssl.org/source/openssl-1.0.2h.tar.gz
cd /soft && tar -xvf openssl-1.0.2h.tar.gz && cd openssl-1.0.2h && ./config --prefix=/usr --openssldir=/usr/local/openssl shared -fPIC && make && make install
verify
openssl
OpenSSL> version
OUTPUT
OpenSSL 1.0.2h
apache service dependency \\
yum install pcre pcre-devel -y
Set Timezone
====== remove localtime ======
sudo rm /etc/localtime
sudo ln -s /usr/share/zoneinfo/America/Los_Angeles /etc/localtime
Now reboot the machine and verify:
Whether you are able to login without any password?\\
Whether the machine boots properly after the update?\\
Whether selinux was disabled? \\
sestatus
Whether iptables is turn off?\\
service iptables status
===== Install required system software =====
==== Install Apache ====
cd /soft
tar xvf httpd-2.2.31.tar.gz
cd httpd-2.2.31
./configure --prefix=/opt/apache2 --enable-mods-shared=most --enable-ssl --enable-usertrack --enable-unique-id --with-included-apr --with-pcre=/usr/bin/pcre-config
make && make install
echo 'export APXS2=/opt/apache2/bin/apxs' >> /etc/profile.d/crossbow.sh
logout
You will need an additional script to ensure this service is started after a system reboot. Look at the section titled [[https://wiki.exphosted.com/doku.php/setup_an_app_server#init_scripts|init scripts]].
== Verify ==
/opt/apache2/bin/apachectl start
lynx localhost
Expected output: \\
You should see a default page text. \\
=== Install Apache modules ===
cd /soft
tar xvf apache-upload-progress-module.tar
cd apache-upload-progress-module
/opt/apache2/bin/apxs -c -i -a mod_upload_progress.c
== Verify: ==
ls /opt/apache2/modules/ | grep mod_upload_progress.so
Expected output:
mod_upload_progress.so
Additionally, repeat the verification steps from Apache Installation section above.
====Setup Ruby Enterprise Edition [EOL] ====
cd /soft
tar -zxvf ruby-enterprise-1.8.7-2011.03.tar.gz
cd ruby-enterprise-1.8.7-2011.03
./installer --dont-install-useful-gems
echo 'pathmunge /opt/ruby-enterprise-1.8.7-2011.03/bin' >> /etc/profile.d/crossbow.sh
ln -s /opt/ruby-enterprise-1.8.7-2011.03/bin/ruby /usr/bin/
ln -s /opt/ruby-enterprise-1.8.7-2011.03/bin/irb /usr/bin/
ln -s /opt/ruby-enterprise-1.8.7-2011.03/bin/gem /usr/bin/
cp /soft/gc_tuned_ruby /opt/ruby-enterprise-1.8.7-2011.03/bin
logout
== Verify ==
ruby -v
Expected output:
ruby 1.8.7 (2011-02-18 patchlevel 334) [x86_64-linux], MBARI 0x6770, Ruby Enterprise Edition 2011.03
=== Install //correct// version of RubyGems and Passenger ===
cd /soft
tar zxvf rubygems-1.3.7.tgz
cd rubygems-1.3.7
sudo ruby setup.rb
== Verify ==
gem --version
Expected output:
1.3.7
==== Rake 0.8.7 ====
We are removing the shipped version and installing an older version.
#gem uninstall rake
gem install rake -v 0.8.7
=== Install Passenger ===
gem install passenger -v "3.0.9"
== Verify ==
gem list | grep passenger
Expected output:
passenger (3.0.9)
#It has been observed (but not throughly test) that the app works fine with -v #"4.0.9" as well
=== Integrate Apache and Passenger ===
yum install httpd-devel
yum install apr-devel
yum install apr-util-devel
yum install curl-devel
/opt/ruby-enterprise-1.8.7-2011.03/bin/passenger-install-apache2-module
You will see a CUI. Keep on pressing next with a space bar and it will install the required module.
== Verify ==
This will be automatically verified in the next section.
==== Setup Apache Conf files ====
cd /opt/apache2/conf
cp /soft/httpd.conf .
== Verify ==
IP addresses in the conf file - By Default, Apache listens on All IP addresses. \\
Additionally, repeat the verification steps from Apache Installation section above. \\
Make sure the RailsEnv variable is correctly set. \\
==== Install FFMPEG ====
cd /soft
tar xvf ffmpeg-1.2.4-working.tar.gz
cd ffmpeg-1.2.4
cd faac-1.28
./bootstrap
./configure --enable-shared
make
make install
cd ../lame-3.99.5
./configure --enable-shared
make
make install
ldconfig
cd ../libogg-1.3.1
./configure
autoreconf --force --install
make && make install
cd ../speex-1.2rc1
./configure
make
make install
yum install libogg-devel yasm yasm-devel
cd ../libtheora-1.1.1
./configure --enable-shared
make
make install
make check
cd ../libvorbis-1.3.3
./configure
make
make install
cd ../libvpx/
./configure
make
make install
make && make install
cd ../x264/
./configure --enable-shared --disable-asm
make
make fprofiled
make install
cd ../
./configure --datadir=/usr/share/ffmpeg --extra-cflags=-I/tmp/ffmpeg-1.2.4/include --extra-ldflags=-L/tmp/ffmpeg-1.2.4/lib --bindir=/usr/local/bin --extra-libs=-ldl --enable-gpl --enable-nonfree --enable-libmp3lame --enable-libvorbis --enable-libvpx --enable-libx264 --enable-libtheora --enable-libspeex --enable-libfaac
make
make install
create symlink for dependency ffmpeg \\
ln -s /usr/local/lib/libx264.so.140 /usr/lib64/libx264.so.140
ln -s /usr/local/lib/libtheoraenc.so.1 /usr/lib64/libtheoraenc.so.1
ln -s /usr/local/lib/libtheoradec.so.1 /usr/lib64/libtheoradec.so.1
ln -s /usr/local/lib/libspeex.so.1 /usr/lib64/libspeex.so.1
ln -s /usr/local/lib/libmp3lame.so.0 /usr/lib64/libmp3lame.so.0
ln -s /usr/local/lib/libmp3lame.so.0 /usr/lib64/libmp3lame.so.0
ln -s /usr/local/lib/libfaac.so.0 /usr/lib64/libfaac.so.0
ln -s /usr/local/lib/libfaac.so.0 /usr/lib64/libfaac.so.0
FFMPEG will be installed at /usr/local/bin. \\
Additionally, the ruby ffmpeg wrapper we use streamio-ffmpeg needs ffmpeg at /usr/bin. \\
ln -nfs /usr/local/bin/ffmpeg /usr/bin/ffmpeg
If FFMPEG shows this error "ffmpeg: error while loading shared libraries: *.so*: cannot open shared object file: No such file or directory"
Run this command;
echo "/usr/local/lib" >> /etc/ld.so.conf
ldconfig
Verify \\
Expected output: \\
ffmpeg version 1.2.4 Copyright (c) 2000-2013 the FFmpeg developers
built on Jan 7 2014 01:23:25 with gcc 4.1.2 (GCC) 20080704 (Red Hat 4.1.2-54)
configuration: --datadir=/usr/share/ffmpeg --extra-cflags=-I/tmp/ffmpeg-1.2.4/include --extra-ldflags=-L/tmp/ffmpeg-1.2.4/lib --bindir=/usr/local/bin --extra-libs=-ldl --enable-gpl --enable-nonfree --enable-libmp3lame --enable-libvorbis --enable-libvpx --enable-libx264 --enable-libtheora --enable-libspeex --enable-libfaac
libavutil 52. 18.100 / 52. 18.100
libavcodec 54. 92.100 / 54. 92.100
libavformat 54. 63.104 / 54. 63.104
libavdevice 54. 3.103 / 54. 3.103
libavfilter 3. 42.103 / 3. 42.103
libswscale 2. 2.100 / 2. 2.100
libswresample 0. 17.102 / 0. 17.102
libpostproc 52. 2.100 / 52. 2.100
Hyper fast Audio and Video encoder
usage: ffmpeg [options] [[infile options] -i infile]... {[outfile options] outfile}...
Use -h to get full help or, even better, run 'man ffmpeg'
==== Install JRE ====
cd /soft
yum localinstall jre-7u45-linux-x64.rpm
== Verify ==
java -version
Expected output:
java version "1.7.0_45"
Java(TM) SE Runtime Environment (build 1.7.0_45-b18)
Java HotSpot(TM) 64-Bit Server VM (build 24.45-b08, mixed mode)
==== Install OpenOffice ====
* Check the release specific version required on https://wiki.exphosted.com/doku.php/releasespecifictasks under Predeployment tasks.
cd /soft
tar zxvf Apache_OpenOffice_4.0.1_Linux_x86-64_install-rpm_en-US.tar.gz
cd en-US/RPMS
yum localinstall *.rpm --nogpgcheck
/opt/openoffice4/program/soffice -headless -accept="socket,host=127.0.0.1,port=8100;urp;" -nofirststartwizard&
== Verify ==
netstat -anp | grep 8100
Expected output:
tcp 0 0 127.0.0.1:8100 0.0.0.0:* LISTEN 15465/soffice.bin
==== Install Pdftk 1.44 ====
cd /soft
yum localinstall pdftk-1.44-2.el6.rf.x86_64.rpm
Yum will also install dependency packages:
(1/6): java-1.4.2-gcj-compat-1.4.2.0-40jpp.115.x86_64.rpm
(2/6): jpackage-utils-1.7.3-1jpp.3.el5.noarch.rpm
(3/6): gjdoc-0.7.7-12.el5.x86_64.rpm
(4/6): antlr-2.7.6-4jpp.2.x86_64.rpm
(5/6): pdftk-1.44-2.el5.rf.x86_64.rpm
(6/6): libgcj-4.1.2-54.el5.x86_64.rpm
== Verify ==
pdftk --version
Expected output:
pdftk 1.44 a Handy Tool for Manipulating PDF Documents
Copyright (C) 2003-10, Sid Steward - Please Visit: www.pdftk.com
This is free software; see the source code for copying conditions. There is
NO warranty, not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
==== Install PDF2JSON ====
cd /soft
mkdir pdf2json
mv pdf2json-0.61.tar.gz pdf2json
cd pdf2json
tar zxvf pdf2json-0.61.tar.gz
make && make install
== Verify ==
pdf2json
Expected output:
pdf2json version 0.61 http://flexpaper.devaldi.com/pdf2json/, based on Xpdf version 3.02
==== Install SWFTOOLS ====
Install SWFTOOLS through source code or RPM
=== PDF2SWF Dependencies ===
#######################
FROM SOURCE CODE
#######################
You need freetype and jpeglib to compile pdf2swf.
Links:
freetype:
http://www.freetype.org
jpeglib:
http://www.ijg.org/files/jpegsrc.v6b.tar.gz
FREETYPE LIB
cd /soft
tar xvzf freetype-2.6.tar.gz
cd freetype-2.6
./configure && make && make install
JPEG LIB
cd /soft
wget "http://www.ijg.org/files/jpegsrc.v6b.tar.gz"
tar xvzf jpegsrc.v6b.tar.gz
cd jpeg-6b
ln -s /usr/bin/libtool libtool
./configure --prefix=/usr --enable-static --enable-shared
cp /usr/lib64/libjpeg.so.62.0.0 /usr/lib64/libjpeg.so.62.0.0.old
cd .libs
cp -f libjpeg.so.62.0.0 /usr/lib64/
Install SWFTOOLS
cd /soft
wget "http://www.swftools.org/swftools-0.9.2.tar.gz"
tar -zvxf swftools-0.9.2.tar
cd swftools-0.9.2
./configure
make
make install
##############################
RPM METHOD
##############################
PDF2SWF program requires //libgif// and //freetype// headers. \\
Its recommended to replace libjpeg-turbo with libjpeg-6b instead as swftools depend on it.
libjpeg-turbo-1.2.1-3.el6_5.x86_64
libjpeg-turbo-devel-1.2.1-3.el6_5.x86_64
In-case you in confusion, revert to turbo version by
yum reinstall libjpeg-turbo-1.2.1-3.el6_5.x86_64 libjpeg-turbo-devel-1.2.1-3.el6_5.x86_64 -y
yum install giflib-devel freetype-devel
=== Compile and Install ===
yum install giflib-devel freetype-devel
wget ftp://ftp.pbone.net/mirror/ftp5.gwdg.de/pub/opensuse/repositories/home:/Kris_Shannon:/RHEL_or_CentOS_Extras/CentOS_6/x86_64/swftools-0.9.2-3.1.el6.sis.x86_64.rpm
rpm -ivh swftools-0.9.2-3.1.el6.sis.x86_64.rpm
Verify
pdf2swf -V
Expected output:
pdf2swf - part of swftools 0.9.2
== Verify ==
pdf2swf -V
Expected output:
pdf2swf - part of swftools 0.9.2
==== Install MUPDF ====
cd /soft
unzip mupdf-1.2-source.zip
cd mupdf-1.2-source
make && make install
===Verify===
mupdf
Expected output:
usage: mupdf [options] file.pdf [page]
-b - set anti-aliasing quality in bits (0=off, 8=best)
-p - password
-r - resolution
==== Install Microsoft Core Fonts =====
cd soft
yum install ImageMagick-6.5.4.7-7.el6_5.x86_64.rpm
yum install ImageMagick-devel-6.5.4.7-7.el6_5.x86_64.rpm
rpm -ivh msttcorefonts-2.5-1.noarch.rpm
cd /usr/share/fonts/default/
ln -s ../msttcorefonts TrueType
== Verify ==
This package only installs font files. You will not be able to install rmagic gem if this was unsuccesfull.
==== Install Subversion and ImageMagick====
install Imagemagic through source code or rpm
yum install subversion -y
yum install pcre-devel ImageMagick ImageMagick-devel git mutt gcc gcc-c++ libxml2 libxml2-devel mysql-devel make -y
rpm -qa Image*
ImageMagick-6.5.4.7-7.el6_5.x86_64
ImageMagick-devel-6.5.4.7-7.el6_5.x86_64
#If rmagic installation failes (centos 6.7) try this
yum install libtool-ltdl.x86_64 libtool-ltdl-devel.x86_64 -y
wget ftp://ftp.imagemagick.org/pub/ImageMagick/ImageMagick.tar.gz
tar xvzf ImageMagick.tar.gz
cd ImageMagick-
./configure --prefix=/usr/local --libdir=/usr/lib64 --with-libdir=lib64
or
export X_CFLAGS='-I/usr/X11R6/include'
export X_PRE_LIBS='-lSM -lICE'
export X_LIBS='-L/usr/X11R6/lib'
./configure --enable-shared=yes --enable-static=no --with-modules=yes --with-gnu-ld=yes --enable-lzw=yes --with-quantum-depth=8 --with-bzlib=yes --with-dps=yes --with-fpx=yes --with-ttf=yes --with-gs-font-dir=default --with-gslib=no --with-dot=yes --with-jbig=yes --with-jpeg=yes --with-jp2=yes --with-lcms=yes --with-magick-plus-plus=no --with-perl=no --with-png=yes --with-tiff=yes --with-windows-font-dir=/usr/share/fonts/default/ --with-wmf=yes --with-x=yes --with-xml=yes --with-zlib=yes
make && make install
===== Ruby Configuration & Depedencies======
==== Rmagick 1.15.17====
gem install rmagick -v 1.15.17
cd /opt/ruby-enterprise-1.8.7-2011.03/lib/ruby/gems/1.8/gems/rmagick-1.15.17/
make && make install
cd /opt/ruby-enterprise-1.8.7-2011.03/share/RMagick/
setfacl -R -m d:u::7 .
setfacl -R -m u::7 .
setfacl -R -m d:u:expprodl:7 .
setfacl -R -m u:expprodl:7 .
==== Rack 1.3.2 ====
We are removing the shipped version and installing an older version.
gem uninstall rack
gem install rack -v "1.3.2"
==== Bundler 1.0.21 ====
gem install bundler -v "1.0.21"
ln -nfs /opt/ruby-enterprise-1.8.7-2011.03/bin/bundle /usr/local/bin/bundle
==== God & Daemon ====
gem install god -v 0.11.0
gem install daemons
==== God & Daemon ====
gem install highline -v 1.6.2
gem install capistrano -v '2.5.21'
gem install capistrano-ext -v '1.2.1'
Installing Solr/Tomcat On Search Server
wget http://apache.mirrors.hoobly.com/tomcat/tomcat-6/v6.0.32/bin/apache-tomcat-6.0.32.tar.gz
tar -xzf apache-tomcat-6.0.32.tar.gz
mv apache-tomcat-6.0.32 /opt/tomcat6
wget http://mirror.atlanticmetro.net/apache//lucene/solr/1.4.1/apache-solr-1.4.1.tgz
tar -xzf apache-solr-1.4.1.tgz
cd apache-solr-1.4.1
cp -r example/solr /opt/
cp dist/apache-solr-1.4.1.war /opt/solr/
cp dist/apache-solr-1.4.1.war /opt/tomcat6/webapps/solr.war
###Set solr data path in /opt/solr/conf/solrconfig.xml (set to the shared drive on search server, unsure if this is required)
mkdir -p /nfs00/share00/solr/data
###/opt/tomcat6/bin/catalina.sh (set JAVA_OPT under the while loop)
JAVA_OPTS="$JAVA_OPTS -Dsolr.solr.home=/opt/solr"
export JAVA_OPTS
##/opt/tomcat6/conf/tomcat-users.xml to include user/passwd
ADD lines below
##/opt/tomcat6/conf/server.xml, add URIEncoding=“UTF-8”
##Copy schema.xml from /deploy/crossbow/current/solr/conf (src = App server) to /opt/solr/conf (destination = Tomcat/Search server)
##Change ownership of /opt/solr and /opt/tomcat6 to the app user used to run tomcat.
===== Monitoring =====
We use Zabbix and Newrelic for monitoring.
https://wiki.exphosted.com/doku.php/na3_monitoring
===== Init Scripts =====
==== Install God init.d ====
sudo mkdir /var/run/god
sudo chown -R expprodl:expprodl /var/run/god
Install INIT file
cd /etc/init.d
wget --user=expbbbu --password= ftp://qacollab01.exphosted.com/softrepo/app/conf/init/god
Modify the environment for this server (if needed) in the CONFIGFILE.
==== INIT for Apache, God & Zabbix Agent ====
Put the following block in /etc/rc.local
echo "Starting Apache"
/opt/apache2/bin/apachectl start
echo "Starting God"
su - expprodl -s "/deploy/systasks/god.sh start"
echo "Starting ZabbixAgent"
zabbixBin=`which zabbix_agentd`
exec zabbixBin
#for haproxy
/opt/ruby-enterprise-1.8.7-2011.03/bin/ruby /opt/ruby-enterprise-1.8.7-2011.03/bin/remote_syslog --hostname 10-166-152-16
==== Server services ====
chkconfig sendmail on
Other INIT files are in the Application Setup.
====== Application Setup ======
===== Download Precompiled binaries and scripts =====
cd /home/expprodl
cp /hdfs/my.cluster.com/backup/bkup.zip .
unzip bkup.zip
cd bkup
grep -rl 10.229.94. *
grep -rl "www.na3.learnexa.com" *
change those ip , hostnames in conf files according to server setup
move the contents into /home/expprodl/
==== Install crossbow application root skeleton ====
cp /soft/deploy.zip /
cd /
unzip deploy.zip
cd /deploy
grep -rl 10.166.152. *
grep -rl "www.learnexa.com" *
change those ip , hostnames in conf files according to server setup
chown -R expprodl.expprodl /deploy
rm deploy.zip
or create dir using /home/expprodl/crossbow/runcap.sh
choose create directory option
copy systask folder to /deploy/systask
then
rm -rf /deploy/crossbow/shared/assets && ln -s /hdfs/my.cluster.com/crossbow/assets/ /deploy/crossbow/shared/assets
rm -rf /deploy/crossbow/shared/log && ln -s /hdfs/my.cluster.com/crossbow/log /deploy/crossbow/shared/log
rm -rf /deploy/crossbow/shared/streams/ && ln -s /hdfs/my.cluster.com/crossbow/streams/ /deploy/crossbow/shared/streams/
rm -rf /deploy/crossbow/shared/system/ && ln -s /hdfs/my.cluster.com/crossbow/shared/system/ /deploy/crossbow/shared/system/
rm -rf /deploy/crossbow/shared/assets && ln -s /hdfs/my.cluster.com/crossbow/assets/protected /deploy/crossbow/shared/assets
Then run deploy:cold
cd /soft
rpm -ivvh memcached-1.4.5-1.el5.x86_64.rpm
Modify the configuration so memcached allows network clients to connect:
edit /etc/sysconfig/memcached
OPTIONS="-l 0.0.0.0"
Memcached is also monitored by God.
cd /soft
tar zxvf node-v0.2.4.tar.gz
cd node-v0.2.4
./configure
make && make install
/home/expprodl/bin/chat_sss.sh start
add below line in /etc/rc.local
runuser -l expprodl -c '/bin/sh /home/expprodl/bin/chat_sss.sh start '
runuser -l expprodl -c '/bin/sh /deploy/systasks/god.sh start'
/bin/sh /opt/apache2/bin/apachectl start
change uid & gid of the appuser
vim /etc/passwd
expprodl:x:2000:2000::/home/expprodl:/bin/bash
vim /etc/group
expprodl:x:2000:
chown -R expprodl:expprodl /deploy/
chown -R expprodl:expprodl /home/expprodl/
chown -R expprodl:expprodl /opt/solr/
chown -R expprodl:expprodl /opt/tomcat6/
chown expprodl:expprodl /etc/init.d/god
chown -R expprodl:expprodl /var/run/god
chown -R expprodl:expprodl /soft/
chown -R expprodl:expprodl /opt/openoffice4/
chown -R expprodl:expprodl /opt/ruby*/
====== Detailed Log ======
0.2 - added development libraries required for compiling native gem dependencies - gcc gcc-c++ libxml2-devel mysql-devel libxslt-devel make
0.1 - created.
###end
{{:na3.png|}}
===== Meeting server =====
follow link https://wiki.exphosted.com/doku.php/setup_bbb_081_expertusone
====== Mysql Db Server ======
adduser expprodl
Mysql slave server roles
solr +tomcat + slavedb
Mysql command
CREATE USER 'dev'@'localhost' IDENTIFIED BY 'devpassword';
CREATE USER 'dev'@'%' IDENTIFIED BY 'devpassword'; ## % will allow to login from any ip (or) You can give specific ip access
create database databasename DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci;
grant all on databasename.* to username@'localhost' identified by 'password';
grant all on databasename.* to username@'%' identified by 'password';
Installing Solr/Tomcat On Search Server
Install the /home/expprodl/bin folder
wget --user=expbbbu --password= ftp://qacollab01.exphosted.com/softrepo/app/deploystructure/deploy.zip
unzip deploy.zip
chown -hR expprodl.expprodl deploy
rm deploy.zip
wget https://archive.apache.org/dist/tomcat/tomcat-6/v6.0.32/bin/apache-tomcat-6.0.32.tar.gz
tar -xzf apache-tomcat-6.0.32.tar.gz
mv apache-tomcat-6.0.32 /opt/tomcat6
wget https://archive.apache.org/dist/lucene/solr/1.4.1/apache-solr-1.4.1.tgz
tar -xzf apache-solr-1.4.1.tgz
cd apache-solr-1.4.1
cp -r example/solr /opt/
cp dist/apache-solr-1.4.1.war /opt/solr/
cp dist/apache-solr-1.4.1.war /opt/tomcat6/webapps/solr.war
Download newrelic agent for Java application and extract it to /opt/tomcat6/
###Set solr data path in /opt/solr/conf/solrconfig.xml (set to the shared drive on search server, unsure if this is required)
##/opt/tomcat6/bin/catalina.sh (set JAVA_OPT under the while loop - mostly line #108 below done)
comment out
# ---- New Relic switch automatically added to start command on 2015 Jul 06, 03:40:53
#NR_JAR=/opt/tomcat6/newrelic/newrelic.jar; export NR_JAR
#CATALINA_OPTS="$CATALINA_OPTS -javaagent:$NR_JAR"; export CATALINA_OPTS
and add;
export JAVA_OPTSVA_OPTS="$JAVA_OPTS -Dsolr.solr.home=/opt/solr -server -Xms256m -Xmx1024m -javaagent:/opt/tomcat6/newrelic/newrelic.jar"
if you are not using newrelic then just give this
JAVA_OPTS="$JAVA_OPTS -Dsolr.solr.home=/opt/solr"
export JAVA_OPTS
##/opt/tomcat6/conf/tomcat-users.xml to include user/passwd
ADD lines below
##/opt/tomcat6/conf/server.xml, add URIEncoding=“UTF-8” (mostly line # 69)
##Copy schema.xml from /deploy/crossbow/current/solr/conf (src = App server) to /opt/solr/conf (destination = Tomcat/Search server)
##Change ownership of /opt/solr and /opt/tomcat6 to the app user used to run tomcat.
before install mysql if there is any package already installed remove it . or it may confict error
rpm -qa | grep -i mysql
for more info https://wiki.exphosted.com/doku.php/setup_db_server?s[]=rpm&s[]=mysql
wget http://downloads.mysql.com/archives/mysql-5.5/MySQL-shared-compat-5.5.13-1.rhel5.x86_64.rpm
wget http://downloads.mysql.com/archives/mysql-5.5/MySQL-client-5.5.13-1.rhel5.x86_64.rpm
wget http://downloads.mysql.com/archives/mysql-5.5/MySQL-server-5.5.13-1.rhel5.x86_64.rpm
wget http://downloads.mysql.com/archives/mysql-5.5/MySQL-devel-5.5.13-1.rhel5.x86_64.rpm
ln -s /usr/lib64/mysql/libmysqlclient.a /usr/lib64/libmysqlclient.a
rpm -ivvh MySQL-shared-compat-5.5.13-1.rhel5.x86_64.rpm
rpm -ivvh MySQL-devel-5.5.13-1.rhel5.x86_64.rpm
rpm -ivvh MySQL-client-5.5.13-1.rhel5.x86_64.rpm
rpm -ivvh MySQL-server-5.5.13-1.rhel5.x86_64.rpm
login app server
login as app user = expprodl
cd /deploy/crossbow/current
bundle install
bundle exec rake db:create --trace RAILS_ENV=staging
bundle exec rake db:migrate --trace RAILS_ENV=staging
bundle exec rake db:seed --trace RAILS_ENV=staging
How To Set Up Master Slave Replication in MySQL below link
https://www.digitalocean.com/community/tutorials/how-to-set-up-master-slave-replication-in-mysql
Master Configuration;
[client]
port = 3306
socket = /var/lib/mysql/mysql.sock
[mysqld]
port = 3306
socket = /var/lib/mysql/mysql.sock
user=mysql
server-id=1
skip-external-locking
key_buffer_size = 256M
max_allowed_packet = 16M
tmp_table_size=64M
max_heap_table_size=64M
table_open_cache = 256
table_cache=2048
sort_buffer_size = 1M
read_buffer_size = 1M
read_rnd_buffer_size = 4M
myisam_sort_buffer_size = 64M
thread_cache_size = 16
interactive_timeout=600
wait_timeout=28800
query_cache_size= 16M
max_connections=200
thread_concurrency = 4
log_bin_trust_function_creators=1
tmpdir=/var/lib/mysql/
datadir=/var/lib/mysql/
log_bin=/var/log/mysql/xlogs/cbprd
max_binlog_size=10M
skip-name-resolve
binlog_format=mixed
innodb_buffer_pool_size = 8G
[mysqldump]
quick
max_allowed_packet = 16M
[mysql]
no-auto-rehash
[myisamchk]
key_buffer_size = 128M
sort_buffer_size = 128M
read_buffer = 2M
write_buffer = 2M
[mysqlhotcopy]
interactive-timeout
Slave configuration;
[client]
port = 3306
socket = /var/lib/mysql/mysql.sock
[mysqld]
port = 3306
socket = /var/lib/mysql/mysql.sock
user=mysql
# Disabling symbolic-links is recommended to prevent assorted security risks
symbolic-links=0
#server-id = 2
relay-log-index = slave.relay-bin.index
relay-log = slave-relay-bin
replicate-wild-ignore-table=mysql.%
replicate-wild-ignore-table=information_schema.%
replicate-wild-ignore-table=performance_schema.%
[mysqld_safe]
log-error=/var/log/mysqld.log
pid-file=/var/run/mysqld/mysqld.pid
skip-external-locking
key_buffer_size = 256M
max_allowed_packet = 16M
tmp_table_size=64M
max_heap_table_size=64M
table_open_cache = 256
table_cache=2048
sort_buffer_size = 1M
read_buffer_size = 1M
read_rnd_buffer_size = 4M
myisam_sort_buffer_size = 64M
thread_cache_size = 16
interactive_timeout=600
wait_timeout=28800
query_cache_size= 16M
max_connections=200
thread_concurrency = 4
tmpdir=/var/lib/mysql/tmpdir
datadir=/var/lib/mysql/datadir
server-id = 2
innodb_buffer_pool_size = 8G
[mysqldump]
quick
max_allowed_packet = 16M
[mysql]
no-auto-rehash
[myisamchk]
key_buffer_size = 128M
sort_buffer_size = 128M
read_buffer = 2M
write_buffer = 2M
[mysqld]
skip-name-resolve
[mysqlhotcopy]
interactive-timeout
Make sure that Master server has
runbackup.sh in /home/expprodl/bin/
and
you have /deploy/crossbow/backups folder created else runcap mysql database backup will show error
====== Haproxy Server ======
Roles chat + haproxy
login root 10.229.494.14
cd /soft/haproxy-1.
make TARGET=linux2628 USE_OPENSSL=1 CPU=native USE_ZLIB=1 USE_PCRE=1
make install
adduser haproxy
adduser expprodl
Add the following line on /etc/rsyslog.conf
# Provides UDP syslog reception
$ModLoad imudp
$UDPServerRun 514
$UDPServerAddress 127.0.0.1
local1.* /var/log/haproxy.log
Then restart the rsyslog service "service rsyslog restart"
To verify;
netstat -anu |grep :514
Then Enable log rotate;
/var/log/haproxy.log {
missingok
# hourly
daily
size 1M
copytruncate
create 0600 root root
rotate 1
}
Follow link for LB Chat https://wiki.exphosted.com/doku.php/chat_implementation?s[]=haproxy&s[]=chat
==== Update Kernel parameters ====
https://wiki.exphosted.com/doku.php/haproxy_changes
Below code is for HTTP only.
[root@10-229-94-14 ~]# cat /opt/haproxy/haproxy.cfg
global
daemon
log 127.0.0.1 local1
maxconn 35000
user haproxy
group haproxy
stats socket /var/run/haproxy/haproxy.sock mode 0600 level admin
spread-checks 5
#user
defaults
log global
mode http
# option dontlognull
#option httplog
option abortonclose
#option forwardfor
retries 3
option redispatch
timeout connect 35000ms
timeout client 360000ms
timeout server 360000ms
timeout http-request 25s
#option http-close
#option http-server-close
no option log-health-checks
#stats enable
#stats uri /learnexalb
#stats realm Learnexa\ LB
#stats auth admin:Expertus2091
frontend http-ingress
option httplog
mode http
bind 10.229.94.14:80
# bind 10.229.94.240:80
# bind 10.229.94.241:80
# bind 10.229.94.242:80
# bind 10.229.94.243:80
# bind 10.229.94.244:80
# bind 10.229.94.245:80
# bind 10.229.94.246:80
# bind 10.229.94.247:80
# bind 10.229.94.248:80
# bind 10.229.94.249:80
# bind 10.229.94.250:80
#server web1 www.na3.expertus.com
#server web2 www.na4.expertus.com
acl chatreq url_beg /chatlist
use_backend chatlistURI if chatreq
# acl prm_is_dead nbsrv(learnexa) lt 1
# use_backend learnexa_bkp if prm_is_dead
default_backend learnexa
#errorfile 503 /opt/haproxy/errors/503.http
frontend chat-ingress
option tcplog
mode tcp
#Juggernaut and Node are local
bind 10.229.94.14:8090
# bind 10.229.94.17:8090
# bind 10.229.94.241:8090
# bind 10.229.94.242:8090
# bind 10.229.94.243:8090
# bind 10.229.94.244:8090
# bind 10.229.94.245:8090
# bind 10.229.94.246:8090
# bind 10.229.94.247:8090
# bind 10.229.94.248:8090
# bind 10.229.94.249:8090
# bind 10.229.94.250:8090
default_backend prodchatrole
backend prodchatrole
mode tcp
server prodchatrole 10.229.94.14:8018 check port 8018 maxconn 200
backend chatlistURI
mode http
server chatlistURI 10.229.94.13:80 check port 80 maxconn 200
backend learnexa
balance roundrobin
cookie SERVERID insert indirect
#source 10.229.94.246
source 10.229.94.14
#timeout client 150000ms
timeout connect 35000ms
timeout server 8m
#option httpchk HEAD /about HTTP/1.0
option httpchk HEAD /about HTTP/1.1\r\nHost:\ na3.learnexa.com\r\nUser-Agent:\ HaProxy
# server prodapp03 10.229.94.16:80 check port 80 cookie app3 maxconn 31 inter 5000 rise 2 fall 3
server prodapp01 10.229.94.13:80 check port 80 cookie app1 maxconn 31 inter 5000 rise 2 fall 3
server prodapp02 10.229.94.16:80 check port 80 cookie app2 maxconn 31 inter 5000 rise 2 fall 3
# server prodapp03 10.229.94.20:80 check port 80 cookie app3 maxconn 31 inter 5000 rise 2 fall 3 disabled
# option httpclose
option forwardfor
backend learnexa_bkp
balance roundrobin
cookie SERVERID insert indirect
server prodapp01 10.229.94.13:80 cookie app1 maxconn 64 inter 8000
server prodapp02 10.229.94.16:80 cookie app2 maxconn 64 inter 8000
# server prodapp03 10.229.94.19:80 check port 80 cookie app3 maxconn 31 inter 5000 rise 2 fall 3
#option httpclose
listen stats :9090
mode http
stats enable
stats uri /learnexalb
stats realm Learnexa\ LB
stats auth admin:Expertus2091
#######Haproxy start command
haproxy -f /opt/haproxy/haproxy.cfg
########Haproxy stop kill pid
ps -ef | grep haproxy
haproxy 2943 1 0 May27 ? 00:15:56 haproxy -f /opt/haproxy/haproxy.cfg
kill 2943
Below code is for HTTPS and HTTP (Multi-tenant)
global
nbproc 1
debug
daemon
maxconn 35000
user haproxy
group haproxy
stats socket /var/run/haproxy/haproxy.sock mode 0600 level admin
spread-checks 5
log 127.0.0.1 local1
tune.ssl.default-dh-param 2048
defaults
log global
mode http
option httplog
retries 3
option redispatch
no option log-health-checks
# timeout connect 35000ms
# timeout client 360000ms
# timeout http-request 25s
timeout connect 3500ms
timeout client 36000ms
timeout http-request 28s
timeout tunnel 1h
timeout server 8m
#Ignoring The prefetch error messages
# errorfile 408 /dev/null
frontend sub_site_to_ssl_redirection
mode http
option forwardfor
option http-server-close
bind 10.229.94.14:80
redirect prefix https://na3.learnexa.com code 301
errorfile 503 /opt/haproxy/errors/503.http
frontend multi_tenant
bind 10.229.94.200:80
mode http
option httplog
option httpclose
option forwardfor
option http-server-close
reqadd X-Forwarded-Proto:\ http
default_backend common_www_backend
errorfile 503 /opt/haproxy/errors/503.http
frontend sub_site_secured
bind 10.229.94.14:443 ssl crt /opt/haproxy/ssl/learnexa.pem
mode http
option httplog
option httpclose
option forwardfor
reqadd X-Forwarded-Proto:\ https
default_backend common_www_backend
errorfile 503 /opt/haproxy/errors/503.http
backend common_www_backend
mode http
option httplog
option forwardfor
option httpchk HEAD /about HTTP/1.1\r\nHost:\ na3.learnexa.com\r\nUser-Agent:\ HaProxy
stats enable
stats hide-version
stats uri /learnexalb
stats realm Learnexa\ LB
stats auth admin:Expertus2091
balance roundrobin
cookie SERVERID insert indirect nocache
option forwardfor
# acl chatreq url_beg /chatlist
# use_backend chatlistURI if chatreq
server app1 10.229.94.13:80 cookie server1 weight 1 maxconn 1024 check cookie app1
server app2 10.229.94.16:80 cookie server2 weight 1 maxconn 1024 check cookie app2
frontend chat-ingress
option tcplog
mode tcp
bind 10.229.94.14:8090
bind 10.229.94.200:8090
default_backend prodchatrole
backend prodchatrole
mode tcp
server prodchatrole 10.229.94.14:8018 check port 8018 maxconn 200
listen :80
mode http
option httplog
acl chatreq url_beg /chatlist
use_backend chatlistURI if chatreq
backend chatlistURI
mode http
server chatlistURI 10.229.94.13:80 check port 80 maxconn 200
==Add host entry==
According to setup can change
vi /etc/hosts
10.229.94.13 www.na3.expertus.com
10.229.94.14 www.10.229.94.14
10.229.94.16 www.na4.expertus.com
10.229.94.250 na3.chat.learnexa.com
===== Improving iptables performance =====
===== Background =====
It was found that above steps of using iptables alone for blocking IP's consumed less amount of resource utilization (CPU & Memory) on our Load Balancer.
===== Solution =====
IPSET - IP sets are a framework inside the Linux 2.4.x and 2.6.x kernel, which ensures lightning speed when matching an entry against a set.
-> 1. Store multiple IP addresses or port numbers and match against the collection by iptables at one swoop;
-> 2. Dynamically update iptables rules against IP addresses or ports without performance penalty;
-> 3. Express complex IP address and ports based rulesets with one single iptables rule and benefit from the speed of IP sets
===== Steps Involved =====
STEP : 1
Install the ipset module to the server
yum install ipset
STEP : 2
Configure the IPSET Match statement
ipset create blocklist host:ip
STEP : 3
Configure IPTABLES to match the blocklist
iptables -A INPUT -m set --match-set blocklist src -j DROP
STEP : 4
Add IP's to the IPSET
ipset add blocklist
===== Initial Files Needed =====
/var/log/haproxy.log* - Load Balancer logs
/opt/ipscript/block_ip.sh - The primary script
/opt/ipscript/ip.blacklist - set of ip addresses to be blocked
/opt/ipscript/rules.new - URL's to be blocked
/opt/ipscript/release_ip.sh - Temporarily releases the ip
===== IP Script written for automating the updates =====
## block_ip.sh - Updated by Syed
## This script can be run every minute safely (root user crontab) and is designed to be easy on server resources (CPU and Memory)
## May need further optimization
#/bin/bash
IPSCRIPT=/opt/ipscript
DATE=$(date +"%d-%m-%Y")
LOG=/var/log/
#tail -n 500 /var/log/haproxy.* >> $IPSCRIPT/templog
#TEMP_LOG=$IPSCRIPT/templog
sort -u rules.new >> rules.temp;
rm -f rules.new;
mv -f rules.temp rules.new;
echo -e "\nReading logs, Performing de-dup and Backup. PLEASE WAIT!\n"
while true;do echo -n .;sleep 1;done &
grep -f rules.new $LOG/haproxy.*|awk '{print $6}'|cut -d':' -f1|sort -u >> $IPSCRIPT/ip.blacklist;
#rm -f TEMP_LOG;
sort -u $IPSCRIPT/ip.blacklist >> $IPSCRIPT/ip.blacklist.bckup.$DATE;
sort -u $IPSCRIPT/ip.blacklist >> $IPSCRIPT/ip.temp;
> $IPSCRIPT/ip.blacklist;
cp -f $IPSCRIPT/ip.temp $IPSCRIPT/ip.blacklist;
rm -f $IPSCRIPT/ip.temp;
sort -u $IPSCRIPT/ip.blacklist.bckup.$DATE >> $IPSCRIPT/ip.temp
> $IPSCRIPT/ip.blacklist.bckup.$DATE;
cp -f $IPSCRIPT/ip.temp $IPSCRIPT/ip.blacklist.bckup.$DATE;
rm -f $IPSCRIPT/ip.temp
kill $!; trap 'kill $!' SIGTERM
echo -e "\nDone!";
echo -e "\nFlushing the Entries\n"
iptables -L INPUT --line-numbers|grep -i black|awk '{print $1}' >> /$IPSCRIPT/rulenum
for i in `cat $IPSCRIPT/rulenum`; do iptables -D INPUT $i; done
sleep 1;
rm -f $IPSCRIPT/rulenum
ipset flush blacklist;
ipset destroy blacklist;
echo -e "\nDone!"
echo -e "\nAdding Entries\n"
ipset create blacklist hash:ip;
iptables -A INPUT -m set --match-set blacklist src -j DROP;
while true;do echo -n .;sleep 1;done &
for i in `cat $IPSCRIPT/ip.blacklist`;
do
flock -w 5 /var/lock/iptables -c "ipset add blacklist $i";
done;
sleep 1
kill $!; trap 'kill $!' SIGTERM
echo -e "\nDone!\n";
===== Script to take backup daily =====
## This script can run everyday safely (/etc/cron.daily/)
#/bin/bash
DATE=$(date +"%m-%d-%Y");
#save configuration
ipset save blacklist > /opt/ipscript/ipsetbckup/ipset.conf.$DATE;
===== Script to Remove IP - Ask permission before you do it =====
## release_ip.sh
## You have to remove the entry in the logfile manually first else its useless
#/bin/bash
DATE=$(date +"%d-%m-%Y")
STORE=/opt/ipscript
sed 's/$1//' $STORE/ip.blacklist|sort -u >> $STORE/ip.temp1;
rm -f $STORE/ip.blacklist;
mv -f $STORE/ip.temp1 $STORE/ip.blacklist;
mv -f $STORE/ip.temp1 $STORE/ip.blacklist.bckup.$DATE;
rm -f $STORE/ip.temp1;
ipset del blacklist $1;
===== rules.new =====
## Rules to block unwanted URL's - Last updated 22-JUN-2015 - WARNING - Leave no space at the start of the line
^\/[aA][dD][Mm][Ii][Nn]/fckeditor
[Aa-Zz].php
^\/admin/index.php
^\/administrator/fckeditor/\editor/filemanager/browser/default/connectors/test.html
\/admin\/phpmyadmin\/scripts\/setup.php
\/[cC][gG][iI]
\/[cC][gG][iI][-_][bB][iI][nN]\/*
\/[cC][gG][iI][bB][iI][nN]\/
^\/cgi-mod
^\/cgi-sys
\/common\/fckeditor\/editor\/
\/docs/funcspecs/3.jsp
\/[Ee][Zz]2000
fckeditor
[fF][cC][kK][eE][dD][iI][tT][oO][rR]
\/[aA][dD][Mm][Ii][Nn]/fckeditor
^?[gG][cC][lL][iI][dD]
[Ii][nN][Dd][eE][xX].[pP][hH][pP]
\/index.php/admin
^\/knkn/knk/kn.php$
^\/manager\/html
[mM][u][i][e][b][l][a][c][k]cat
[mM][yY][aA][dD][m<][iI][Nn]/scripts\/setup.php
mousemove.jsp
\/muieblackcat
mwheelIntent.jsp
^\/phph/php/\ph.php
phpmyadmin
^\/phpmyadmin/scripts/setup.php
^\/phpMyAdmin\/scripts\/setup.php
\/phppath*
\/pma/*
^\/pma/scripts/setup.php
[pP][hH][mM][yy][aA][dD][mM][iI][nN]
\/[pP][hH][pP][Mm][yY][aA][dD][mM][iI][Nn]/scripts/setup.php
\/\/[pP][hH][pP][Mm][yY][aA][dD][mM][iI][Nn]/scripts/setup.php
\/[pP][mM][aA]/scripts/setup.php
\/sql/phpmyadmin/*
[sS][eE][tT][uU][pP].[pP][hH][pP]
\/[sS][iI][gG][nN][uU][pP].php
\/[sS][yY][sS][tT][eE][mM]/fckeditor/editor
^\/styles/administrator/index.php
^\/styles/wp-login.php
\/[tT][mM][Uu]nblock.cgi
^w00tw00t.at.blackhats.romanian.anti-sec*
^w00tw00t.at.ISC.SANS*
^\/wp-content
^\/wp-login.php$
\/[wW][0oO][0oO][tT][wW][0oO][0oO][tT].at.ISC.SANS*
\/[wW][oO][rR][Dd][pP][rR][eE][sS][sS]/wp-login.php
\/[wW][pP]-[lL][oO][gG][iI][Nn].php
\/xampp\/*.*
\/yebryizpldede.html
\/tmUnblock.cgi$
\/rom0
\/rom-0
^\/plpl/plp/pl.php$
^\/ejpxopsgc.html$
^\/#E8#81#BD*
.*[.]php$
.*[.]jsp$
^\/dana-na/auth/url_default/welcome.cgi
\/caca2.txt$
\/[Dd]iagnostics.asp
\/[rR]inging.at.your.dorbell!
manager\/html
\/forum\/includes\/templates\/error.tpl
\/app/etc/local.xml
\/magmi/conf/magmi.ini
\/filter/tips
\/site.zip
\/site.tar
\/site.tar.gz
\/learnexa.com.zip
\/learnexa.com.tar
\/learnexa.com.tar.gz
\/root/back.css
\/mail/README
\/rc/README
^\/README
\/roundcubemail/README
\/roundcube/README
\/webmail/README
\/dnscfg.cgi
\/payment/pay.cgi
\/[Oo][Vv][Cc][Gg][Ii]/getnnmdata.exe
===== ip.blacklist =====
## block unwanted IP's - Last updated 30-APR-2015
101.222.232.22
103.15.88.76
103.242.101.145
103.253.113.173
103.253.42.28
103.27.127.238
104.255.68.139
109.200.24.36
110.77.142.109
110.77.159.244
110.77.179.232
110.77.197.247
110.77.240.224
110.89.13.167
111.11.181.245
113.135.195.55
113.161.84.88
113.174.198.171
113.53.252.109
114.215.150.17
114.45.135.248
116.226.38.156
118.26.202.100
119.57.17.20
119.82.224.162
120.195.36.188
121.40.187.101
122.155.197.6
123.30.132.164
123.85.194.121
124.11.134.238
125.227.108.82
125.227.114.55
125.81.189.218
133.18.96.54
140.120.90.187
140.224.94.127
1.53.76.193
157.7.209.93
186.128.34.221
186.9.94.199
187.149.44.172
187.149.70.200
187.151.126.62
187.189.10.175
188.132.193.155
188.132.204.208
188.138.104.206
189.148.172.221
189.152.130.84
189.174.22.197
189.225.134.89
190.157.187.56
192.187.110.98
194.28.6.131
194.6.233.15
199.101.185.133
199.217.115.178
200.109.205.109
200.74.138.189
201.214.57.118
201.246.231.226
201.99.63.27
202.129.59.146
202.46.14.130
203.113.122.174
203.127.84.36
203.146.208.153
203.157.45.75
210.61.165.62
211.137.179.236
211.154.6.139
217.12.202.39
218.241.190.177
218.60.29.198
219.144.200.102
219.232.247.108
220.255.1.140
220.255.1.46
220.255.1.49
220.255.1.73
222.137.23.8
222.186.190.194
222.218.142.137
223.205.74.225
31.185.6.10
37.135.168.43
37.55.237.47
42.119.55.3
46.105.247.175
46.118.117.18
46.17.100.114
46.98.6.107
5.35.253.135
58.153.232.85
58.222.21.43
58.234.140.150
60.249.202.132
69.197.186.210
77.38.132.77
78.135.88.147
79.117.15.33
80.230.102.97
82.146.32.60
84.77.89.185
85.114.142.136
88.147.181.141
88.238.79.197
89.111.180.158
91.194.84.124
91.200.160.148
91.217.90.49
91.235.176.40
92.114.220.166
93.76.61.237
94.23.165.144
===== Performance comparison iptables vs iptables + ipset =====
iptables reload takes more than 2 second for 956 IP's -> Takes more hardware resources - Fast
iptables + ipset takes few seconds to reload 121 IP's -> Takes less hardware resources - Fast
Tested on 10.229.94.14.
With IPSET parsing parallel log file size greater than 10Mb
[root@10-229-94-14 ipscript]# date; sh block_ip.sh; date;
Thu Apr 30 06:32:25 EDT 2015
Thu Apr 30 06:32:51 EDT 2015
[root@10-229-94-14 ipscript]# ipset list |wc -l
122
For 122 IP's it takes 22 - 30 seconds
Main Advantage is the response time for incoming request, compared to netfilter default drop list;
REF : http://daemonkeeper.net/781/mass-blocking-ip-addresses-with-ipset/
==== Ip tables fine tuned performance ====
[root@10-166-152-16 ~]# cat /etc/sysconfig/iptables | grep DROP|wc -l
956
[root@10-166-152-16 ~]# date;/var/log/block-ip.sh;date
Tue May 12 08:23:21 EDT 2015
Saving firewall rules to /etc/sysconfig/iptables: [ OK ]
/var/log/ip.blacklist-updated
Saving firewall rules to /etc/sysconfig/iptables: [ OK ]
Tue May 12 08:23:22 EDT 2015
For 956 ip's it take only 1 seconds by iptables
Script slightly modified .Search entire log file & get blocked banned search patterns
Previously have used grep . Now we are using fgrep
I have created haproxy.log.1 12Mb file size to test
Now script took only 4seconds to execute . Search entire log file & get blocked banned search patterns
[root@10-166-152-16 ~]# du -sh /var/log/haproxy.log*
192K /var/log/haproxy.log
12M /var/log/haproxy.log.1
[root@10-166-152-16 ~]# date;/var/log/block-ip.sh;date
Tue May 12 05:18:36 EDT 2015
Saving firewall rules to /etc/sysconfig/iptables: [ OK ]
/var/log/ip.blacklist-updated
Saving firewall rules to /etc/sysconfig/iptables: [ OK ]
Tue May 12 05:18:40 EDT 2015