es5_clean.sh

#!/bin/bash

apt-get update && DEBIAN_FRONTEND=noninteractive \
    apt-get \
    -o Dpkg::Options::="--force-confnew" \
    --force-yes \
    -fuy \
    dist-upgrade && \
    DEBIAN_FRONTEND=noninteractive \
    apt-get \
    -o Dpkg::Options::="--force-confnew" \
    --force-yes \
    -fuy \
    -t stretch-backports install openjdk-8-jdk-headless net-tools wget uuid-runtime python-pip python-dev libyaml-dev httpie jq gawk less silversearcher-ag

pip install --upgrade pip
pip install elasticsearch-curator
mkdir /root/.curator
touch /root/.curator/curator.yml

elastic_version=5.6.11
elastic_deb=https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${elastic_version}.deb
kibana_deb=https://artifacts.elastic.co/downloads/kibana/kibana-${elastic_version}-amd64.deb

# compute half memtotal gigs
mem_gb=$(awk '/MemTotal/ { printf "%d", $2/1024/1024/2 }' /proc/meminfo)
instance_name=$(http --ignore-stdin --check-status 'http://metadata.google.internal/computeMetadata/v1/instance/name'  "Metadata-Flavor:Google" -p b --pretty none)
# cluster_id=$(uuidgen -r)
cluster_id=$instance_name

(cd /tmp; \
 wget --no-check-certificate $elastic_deb; \
 dpkg -i elasticsearch-${elastic_version}.deb; \
 rm -f elasticsearch-${elastic_version}.deb)

cat <<EOF > /etc/elasticsearch/elasticsearch.yml
cluster.name: ${cluster_id}
network.host: _local_, _eth0_
http.port: 9200
bootstrap.memory_lock: true
repositories.url.allowed_urls: ["https://*.amazonaws.com/*","https://storage.googleapis.com/*"]
reindex.remote.whitelist: 10.*.*.*:*, _local_:*

EOF

cat <<EOF > /etc/elasticsearch/jvm.options
-Xms${mem_gb}g
-Xmx${mem_gb}g
-XX:+UseConcMarkSweepGC
-XX:CMSInitiatingOccupancyFraction=75
-XX:+UseCMSInitiatingOccupancyOnly
-XX:+DisableExplicitGC
-XX:+AlwaysPreTouch
-server
-Xss1m
-Djava.awt.headless=true
-Dfile.encoding=UTF-8
-Djna.nosys=true
-Djdk.io.permissionsUseCanonicalPath=true
-Dio.netty.noUnsafe=true
-Dio.netty.noKeySetOptimization=true
-Dio.netty.recycler.maxCapacityPerThread=0
-Dlog4j.shutdownHookEnabled=false
-Dlog4j2.disable.jmx=true
-Dlog4j.skipJansi=true
-XX:+HeapDumpOnOutOfMemoryError

EOF

cat <<EOF > /etc/security/limits.conf
* soft nofile 65536
* hard nofile 65536
* soft memlock unlimited
* hard memlock unlimited
elasticsearch soft memlock unlimited
elasticsearch hard memlock unlimited

EOF

cat <<EOF > /etc/sysctl.conf
net.ipv4.icmp_echo_ignore_broadcasts = 1
net.ipv4.icmp_ignore_bogus_error_responses = 1
net.ipv4.tcp_syncookies = 1
net.ipv4.conf.all.accept_source_route = 0
net.ipv4.conf.default.accept_source_route = 0
net.ipv4.conf.all.rp_filter = 1
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.all.accept_redirects = 0
net.ipv4.conf.default.accept_redirects = 0
net.ipv4.conf.all.secure_redirects = 0
net.ipv4.conf.default.secure_redirects = 0
net.ipv4.ip_forward = 0
net.ipv4.conf.all.send_redirects = 0
net.ipv4.conf.default.send_redirects = 0
kernel.randomize_va_space = 1
fs.file-max = 65535
kernel.pid_max = 65536
net.ipv4.ip_local_port_range = 2000 65000
net.ipv4.tcp_window_scaling = 1
net.ipv4.tcp_max_syn_backlog = 3240000
net.ipv4.tcp_fin_timeout = 15
net.core.somaxconn = 65535
net.ipv4.tcp_max_tw_buckets = 1440000
net.core.rmem_default = 8388608
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.ipv4.tcp_rmem = 4096 87380 16777216
net.ipv4.tcp_wmem = 4096 65536 16777216
net.ipv4.tcp_congestion_control = cubic
vm.swappiness = 1
net.ipv4.tcp_tw_reuse = 1

EOF

# set all sysctl configurations
sysctl -p

# disable swap
swapoff -a

echo "block/sda/queue/scheduler = noop" >> /etc/sysfs.conf
echo noop > /sys/block/sda/queue/scheduler

sed -i 's/\#LimitMEMLOCK=infinity/LimitMEMLOCK=infinity/g' /usr/lib/systemd/system/elasticsearch.service
sed -i '46iLimitMEMLOCK=infinity' /usr/lib/systemd/system/elasticsearch.service

systemctl daemon-reload
echo install slasticsearch plugins gcs and gce

(cd /usr/share/elasticsearch/bin; \
 ./elasticsearch-plugin install repository-gcs; \
 ./elasticsearch-plugin install analysis-icu; \
 ./elasticsearch-plugin install discovery-gce)

echo start elasticsearch

systemctl enable elasticsearch

# /etc/init.d/elasticsearch start
systemctl start elasticsearch

echo install kibana

(cd /tmp; \
 wget --no-check-certificate $kibana_deb; \
 dpkg -i kibana-${elastic_version}-amd64.deb; \
 rm -f kibana-${elastic_version}-amd64.deb)

cat <<EOF > /etc/kibana/kibana.yml
server.port: 5601
server.host: "localhost"
elasticsearch.url: "http://127.0.0.1:9200"

EOF

echo start kibana

systemctl enable kibana

sleep 3 && systemctl start kibana

echo configure gcs snapshot plugin repository

echo install stackdriver and elasticsearch conf for it

(cd /root && \
    curl -O https://repo.stackdriver.com/stack-install.sh && \
    bash stack-install.sh --write-gcm)

(cd /root && \
    curl -sSO https://dl.google.com/cloudagents/install-logging-agent.sh && \
    sudo bash install-logging-agent.sh)

(cd /opt/stackdriver/collectd/etc/collectd.d/ && \
    curl -O https://raw.githubusercontent.com/Stackdriver/stackdriver-agent-service-configs/master/etc/collectd.d/elasticsearch-1.conf)

service stackdriver-agent restart

echo stackdriver logging for elasticsearch configuration

cat <<EOF > /etc/google-fluentd/config.d/elasticsearch.conf
<source>
  @type tail
  format none
  path /var/log/elasticsearch/es5*.log
  pos_file /var/lib/google-fluentd/pos/elasticsearch.pos
  read_from_head true
  tag elasticsearch
</source>
EOF

service google-fluentd restart

echo done.