====== Linux TCP / kernel network parameters to achieve better haproxy performance ======
===== Current defaults =====
/etc/sysctl.conf\\
net.ipv4.tcp_rmem = 4096 87380 4194304\\
net.ipv4.tcp_wmem = 4096 16384 4194304\\
net.ipv4.tcp_mem = 196608 262144 393216\\
net.ipv4.ip_local_port_range = 32768 61000\\
net.core.netdev_max_backlog = 1000\\
net.core.somaxconn = 128\\
net.core.rmem_default = 129024\\
net.core.wmem_default = 129024\\
net.core.rmem_max = 131071\\
net.core.wmem_max = 131071\\
===== new values =====
/etc/sysctl.conf\\
net.ipv4.tcp_rmem = 4096 87380 8388608\\
net.ipv4.tcp_wmem = 4096 65536 8388608\\
net.ipv4.tcp_mem = 196608 262144 393216 (no change)\\
net.ipv4.ip_local_port_range = 25000 61000\\
net.core.netdev_max_backlog = 2000\\
net.core.somaxconn = 1024\\
net.core.rmem_default = 129024\\
net.core.wmem_default = 129024\\
net.core.rmem_max = 8388608\\
net.core.wmem_max = 8388608\\
sysctl -p\\
/etc/init.d/haproxy restart\\
After kernel update -> current 2.6.18 -> new update for el5 2.6.32 available\\
net.ipv4.tcp_congestion_control=htcp\\
After haproxy upgrade from 1.4.2 -> 1.5.x\\
We can change sticky sessions / session tables parameters\\
=== Open Question ===
Will these changes impact the node and redis (specially redis) anyway ?
A. No, we tested this on our NA3 site and every thing went well.
=== Update ===
# Turn On TCP SYN Cookie Protection
net.ipv4.tcp_syncookies = 1
=== SSL_HAPROXY ===
NA3 setup ;
(Multi-tenant (HTTP) - 162.212.3.238) ---|------------|---|Node1|-------|---------|-------
| HA-Proxy | | | Shared Storage | MySQL|
(Subsite (HTTPS) - 162.212.1.188) ---|------------|---|Node2|-------|---------|-------
The public IP's are nated to internal IP's Haproxy.
162.212.1.188 - 10.229.94.14
162.212.3.238 - 10.229.94.200
na3.mtchat.learnexa.com = Multi-tenant (HTTP eg, expertlearn.com)
na3.chat.learnexa.com = Subsite (HTTPS eg, test.learnexa.com)
We made minor changes to the code.
/deploy/crossbow/current/app/views/layouts/_bottom_script_and_styles.html.haml
24 - chat_server_url = "#{CHAT_CONFIG['juggernaut_protocol']}://#{CHAT_CONFIG['juggernaut_host']}:#{CHAT_CONFIG['ju ggernaut_port']}"
with
24 - p = @controller.request.headers["HTTP_X_MULTI_TENANT"] == "true" ? "http" : "https"
25 - h = @controller.request.headers["HTTP_X_MULTI_TENANT"] == "true" ? "na3.mtchat.learnexa.com" : "na3.chat.learnexa.com"
26 - chat_server_url = "#{p}://#{h}:#{CHAT_CONFIG['juggernaut_port']}"
=== SSL_HAPROXY_CONFIG ===
global
nbproc 1
debug
daemon
maxconn 35000
user haproxy
group haproxy
stats socket /var/run/haproxy/haproxy.sock mode 0600 level admin
spread-checks 5
log 127.0.0.1 local1
tune.ssl.default-dh-param 2048
defaults
log global
mode http
option tcp-smart-accept
option tcp-smart-connect
option tcpka
option srvtcpka
option httplog
option dontlognull
retries 3
option redispatch
no option log-health-checks
timeout connect 10000
timeout client 300000
timeout http-request 28s
timeout tunnel 1h
timeout server 8m
frontend sub_site_to_ssl_redirection
mode http
option forwardfor
option http-server-close
bind 10.229.94.14:80
redirect scheme https if !{ ssl_fc }
default_backend common_www_backend
errorfile 503 /opt/haproxy/errors/503.http
frontend multi_tenant
bind 10.229.94.200:80
mode http
option httplog
option httpclose
option forwardfor
option http-server-close
reqadd X-Forwarded-Proto:\ http
reqadd X-Multi-tenant:\ true
acl chatreq url_beg /chatlist
################################ ADD REDIRECTION MT ########################################################
redirect prefix https://learnexa.exphosted.com code 301 if { hdr(host) -i learnexa.exphosted.com }
############################################################################################################
use_backend chatlistURI if chatreq
default_backend common_www_backend
errorfile 503 /opt/haproxy/errors/503.http
frontend multi_tenant_secure
bind 10.229.94.200:443 ssl crt /opt/haproxy/ssl/certs.d/
mode http
option httplog
option httpclose
option forwardfor
reqadd X-Forwarded-Proto:\ https
acl chatreq url_beg /chatlist
use_backend chatlistURI if chatreq
default_backend common_www_backend
errorfile 503 /opt/haproxy/errors/503.http
frontend sub_site_secured
bind 10.229.94.14:443 ssl crt /opt/haproxy/ssl/certs.d/
mode http
option httplog
option httpclose
option forwardfor
reqadd X-Forwarded-Proto:\ https
# rspadd Strict-Transport-Security:\ max-age=31536000
acl chatreq url_beg /chatlist
use_backend chatlistURI if chatreq
default_backend common_www_backend
errorfile 503 /opt/haproxy/errors/503.http
backend common_www_backend
mode http
option httplog
option forwardfor
option httpchk HEAD /about HTTP/1.1\r\nHost:\ na3.learnexa.com\r\nUser-Agent:\ HaProxy
stats enable
stats hide-version
stats uri /learnexalb
stats realm Learnexa\ LB
stats auth admin:Expertus2091
balance roundrobin
cookie SERVERID insert indirect nocache
option forwardfor
server app1 10.229.94.13:80 cookie server1 weight 1 maxconn 1024 check cookie app1
server app2 10.229.94.16:80 cookie server2 weight 1 maxconn 1024 check cookie app2
frontend chat-ingress
option tcplog
mode tcp
bind 10.229.94.14:8090 ssl crt /opt/haproxy/ssl/certs.d/learnexa.pem
bind 10.229.94.200:8090
default_backend prodchatrole
backend prodchatrole
mode tcp
option forceclose
server prodchatrole 10.229.94.14:8018 check port 8018 maxconn 200
backend chatlistURI
mode http
server chatlistURI 10.229.94.13:80 check port 80 maxconn 200
=== SSL_HAPROXY_CONFIG_RELOAD SCRIPT ===
#/bin/bash
#THIS SCRIPT IS FREE TO USE
#AUTHOR : Syed A. Bahmani
#DESIGNATION : DevOps Engineer
CONF=/opt/haproxy/haproxy.cfg
if [ ! -e $CONF ]
then
echo -e "The file was not found : $CONF \nPlease configure right path in CONF variable!"
exit 1
fi
if [[ $1 == "adddom" || $1 == "deldom" ]]
then
if [[ $1 == "adddom" ]]
then
sed -i "51i redirect prefix https://$2 code 301 if { hdr(host) -i $2 }" $CONF
service haproxy reload 2>1
elif [[ $1 == "deldom" ]]
then
o=$(grep -ni $2 $CONF |cut -d':' -f1)
sed -i "$o d" $CONF
service haproxy reload 2>1
fi
else
echo -e "\nUSEAGE : $(basename $0) adddom|deldom \n"
echo -e "\n++++++++HERE IS THE LIST OF DOMAINS++++++++\n"
grep -i "redirect prefix" $CONF |cut -d' ' -f11
echo -e "\n"
fi
=== SSL_HAPROXY_UPGRADE_FROM_1.4.x_to_1.5.x ===
Upgrade scenario;
#!/bin/bash
if [ ! -d /opt/haproxy/src/ ]
then
mkdir -p /opt/haproxy/src && cd /opt/haproxy/src/
fi
cd /opt/haproxy/src/ && rm -rf *
mkdir -p /opt/haproxy/src/openssl
export STATICLIBSSL=/opt/haproxy/src/openssl/
yum install -y pcre-devel pcre
yum install epel-release -y
yum install httpd-devel apr-devel apr-util-devel curl-devel -y
yum groupinstall "Development Tools" "Additional Development" -y
if [ ! -d /opt/haproxy/src/openssl-1.0.2d ]
then
wget -c "http://openssl.org/source/openssl-1.0.2d.tar.gz" -O /opt/haproxy/src/openssl-1.0.2d.tar.gz
cd /opt/haproxy/src/ && tar xvzf openssl-1.0.2d.tar.gz && cd /opt/haproxy/src/openssl-1.0.2d/
./config --prefix=$STATICLIBSSL no-shared && make && make install_sw
fi
if [ ! -d /opt/haproxy/src/haproxy-1.5.15 ]
then
wget -c "http://www.haproxy.org/download/1.5/src/haproxy-1.5.15.tar.gz" -O /opt/haproxy/src/haproxy-1.5.15.tar.gz
cd /opt/haproxy/src/ && tar xvzf haproxy-1.5.15.tar.gz && cd /opt/haproxy/src/haproxy-1.5.15
make TARGET=linux2628 CPU=native USE_ZLIB=1 USE_PCRE=1 USE_OPENSSL=1 ADDINC=-I$STATICLIBSSL/include ADDLIB="-L$STATICLIBSSL/lib"
make install
rm -rf /opt/haproxy/src/haproxy-1.5.15 && rm -rf /opt/haproxy/src/openssl-1.0.2d
fi
Roll-back scenario
#!/bin/bash
yum install -y pcre-devel pcre
yum install epel-release -y
yum install httpd-devel apr-devel apr-util-devel curl-devel -y
yum groupinstall "Development Tools" "Additional Development" -y
if [ ! -d /opt/haproxy/src/haproxy-1.4.26 ]
then
mkdir -p /opt/haproxy/src && cd /opt/haproxy/src/ && rm -rf * &>/dev/null
wget -c "http://www.haproxy.org/download/1.4/src/haproxy-1.4.26.tar.gz" -O haproxy-1.4.26.tar.gz && tar xvzf haproxy-1.4.26.tar.gz
cd haproxy-1.4.26
make TARGET=linux2628 CPU=native USE_ZLIB=1 USE_PCRE=1
make install
else
rm -rf /opt/haproxy/src/haproxy-1.4.26 && cd /opt/haproxy/src
wget -c "http://www.haproxy.org/download/1.4/src/haproxy-1.4.26.tar.gz" -O haproxy-1.4.26.tar.gz && tar xvzf haproxy-1.4.26.tar.gz
cd haproxy-1.4.26
make TARGET=linux2628 CPU=native USE_ZLIB=1 USE_PCRE=1
make install
fi