Merge pull request #23 from rubygems/es-logstash

Tweak ElasticSearch for logstash, set up logstash 1.5.0
rubygems · Jun 28, 2015 · 419312e · 419312e
2 parents e130a73 + 837ff1f
commit 419312e
Showing 8 changed files with 96 additions and 11 deletions.
diff --git a/Berksfile.lock b/Berksfile.lock
@@ -289,7 +289,7 @@ GRAPH
     dns (>= 0.0.0)
     hostname (>= 0.0.0)
   rubygems-hosts (0.0.10)
-  rubygems-logging (0.0.75)
+  rubygems-logging (0.0.76)
     chef-vault (>= 0.0.0)
     elasticsearch (>= 0.0.0)
     java (>= 0.0.0)

diff --git a/cookbooks/rubygems-logging/metadata.rb b/cookbooks/rubygems-logging/metadata.rb
@@ -1,7 +1,7 @@
 name 'rubygems-logging'
 maintainer 'RubyGems.org ops team'
 
-version '0.0.75'
+version '0.0.76'
 
 depends 'chef-vault'
 depends 'elasticsearch'

diff --git a/cookbooks/rubygems-logging/recipes/server.rb b/cookbooks/rubygems-logging/recipes/server.rb
@@ -3,8 +3,13 @@
 # Recipe:: server
 #
 
-node.default['java']['install_flavor'] = 'openjdk'
-node.default['java']['jdk_version'] = '7'
+node.default['java']['jdk_version'] = '8'
+node.default['java']['install_flavor'] = 'oracle'
+node.default['java']['oracle']['accept_oracle_download_terms'] = true
+
+node.default['java']['jdk']['8']['x86_64']['url'] = 'http://download.oracle.com/otn-pub/java/jdk/8u45-b14/jdk-8u45-linux-x64.tar.gz'
+node.default['java']['jdk']['8']['x86_64']['checksum'] = '1ad9a5be748fb75b31cd3bd3aa339cac'
+
 include_recipe 'java'
 
 include_recipe 'rubygems-logging::server_elasticsearch'

diff --git a/cookbooks/rubygems-logging/recipes/server_elasticsearch.rb b/cookbooks/rubygems-logging/recipes/server_elasticsearch.rb
@@ -7,9 +7,20 @@
 node.default['elasticsearch']['cloud']['aws']['access_key'] = aws_credentials['access_key_id']
 node.default['elasticsearch']['cloud']['aws']['secret_key'] = aws_credentials['secret_access_key']
 node.default['elasticsearch']['cloud']['aws']['region'] = 'us-west-2'
-node.default['elasticsearch']['version'] = '1.5.0'
-node.default['elasticsearch']['allocated_memory'] = '4000m'
-node.default['elasticsearch']['bootstrap']['mlockall'] = true
+
+node.default['elasticsearch']['version'] = '1.6.0'
+
+node.default['elasticsearch']['plugins'] = {
+  'lmenezes/elasticsearch-kopf' => {}
+}
+
+node.default['elasticsearch']['allocated_memory'] = "#{(node['memory']['total'].to_i * 0.5).floor / 1024}m"
+
+node.default['elasticsearch']['mlockall'] = node['memory']['total'].to_i >= 1048576
+
+# Allow recoveries speeds up to 500mb/sec in 12 streams
+node.default['elasticsearch']['indices.recovery.concurrent_streams'] = 12
+node.default['elasticsearch']['indices.recovery.max_bytes_per_sec'] = '500mb'
 
 # node.default['elasticsearch']['custom_config']['threadpool.search.type'] = 'fixed'
 # node.default['elasticsearch']['custom_config']['threadpool.search.size'] = '20'
@@ -35,6 +46,15 @@
 node.default['elasticsearch']['custom_config']['index.refresh_interval'] = '30s'
 node.default['elasticsearch']['custom_config']['index.translog.flush_threshold_ops'] = '50000'
 
+# Bump shard reallocation throttling/limits
+node.default['elasticsearch']['custom_config']['cluster.routing.allocation.node_concurrent_recoveries'] = '100'
+node.default['elasticsearch']['custom_config']['cluster.routing.allocation.cluster_concurrent_rebalance'] = '100'
+
+node.default['elasticsearch']['custom_config']['index.analysis.analyzer.default.type'] = 'keyword'
+node.default['elasticsearch']['custom_config']['index.analysis.analyzer.default.stopwords'] = '_none_'
+
+node.default['elasticsearch']['skip_restart'] = true
+
 node.default['elasticsearch']['gc_settings'] = <<-CONFIG
   -XX:+UseG1GC
   -XX:+HeapDumpOnOutOfMemoryError
@@ -75,8 +95,7 @@
 include_recipe 'elasticsearch'
 include_recipe 'elasticsearch::ebs'
 include_recipe 'elasticsearch::data'
-
-install_plugin 'lmenezes/elasticsearch-kopf'
+include_recipe 'elasticsearch::plugins'
 
 easy_install_package 'elasticsearch-curator'
 

diff --git a/cookbooks/rubygems-logging/recipes/server_logstash.rb b/cookbooks/rubygems-logging/recipes/server_logstash.rb
@@ -29,7 +29,40 @@
   notifies :restart, 'logstash_service[server]', :delayed
 end
 
-include_recipe 'logstash::server'
+node.default['logstash']['instance']['server']['version']            = '1.5.1'
+node.default['logstash']['instance']['server']['source_url']         = 'https://download.elasticsearch.org/logstash/logstash/logstash-1.5.1.tar.gz'
+node.default['logstash']['instance']['server']['checksum']           = 'a12f91bc87f6cd8f1b481c9e9d0370a650b2c36fdc6a656785ef883cb1002894'
+node.default['logstash']['instance']['server']['gc_opts']            = <<-CONFIG
+  -XX:+UseG1GC
+  -XX:+HeapDumpOnOutOfMemoryError
+CONFIG
+node.default['logstash']['instance']['server']['java_home']          = '/usr/lib/jvm/java-8-oracle-amd64'
+
+name = 'server'
+logstash_instance name do
+  action 'create'
+end
+logstash_service name do
+  action [:enable, :start]
+  templates_cookbook 'rubygems-logging'
+end
+logstash_pattern name do
+  action 'create'
+end
+logstash_curator name do
+  action 'create'
+end
+
+# Since logstash 1.5.0, outputs/inputs/filters are separate plugins
+%w(
+  logstash-filter-mutate logstash-filter-grok logstash-filter-date logstash-filter-geoip logstash-output-elasticsearch
+  logstash-input-lumberjack logstash-input-syslog
+).each do |plugin|
+  logstash_plugins plugin do
+    action 'create'
+    instance name
+  end
+end
 
 custom_templates = {
   'inputs'               => 'logstash/inputs.conf.erb',

diff --git a/cookbooks/rubygems-logging/templates/default/logstash/output_elasticsearch.conf.erb b/cookbooks/rubygems-logging/templates/default/logstash/output_elasticsearch.conf.erb
@@ -1,5 +1,6 @@
 output {
   elasticsearch {
-    host => "127.0.0.1"
+    host => [ "127.0.0.1" ]
+    protocol => "http"
   }
 }
diff --git a/cookbooks/rubygems-logging/templates/default/sv-logstash-log-run.erb b/cookbooks/rubygems-logging/templates/default/sv-logstash-log-run.erb
@@ -0,0 +1,2 @@
+#!/bin/sh
+exec svlogd -tt ./main
diff --git a/cookbooks/rubygems-logging/templates/default/sv-logstash-run.erb b/cookbooks/rubygems-logging/templates/default/sv-logstash-run.erb
@@ -0,0 +1,25 @@
+#!/bin/sh
+
+ulimit -Hn 65550
+ulimit -Sn 65550
+
+cd /<%= @options[:home] %>
+exec 2>&1
+# Need to set LOGSTASH_HOME and HOME so sincedb will work
+export LS_HEAP_SIZE=<%= @options[:max_heap] %>
+export LOGSTASH_HOME="<%= @options[:home] %>"
+export GC_OPTS="<%= @options[:gc_opts] %>"
+export JAVA_OPTS="-server -Xms<%= @options[:min_heap] %> -Xmx<%= @options[:max_heap] %> -Djava.io.tmpdir=$LOGSTASH_HOME/tmp/ <%= @options[:java_opts] %> <%= '-Djava.net.preferIPv4Stack=true' if @options[:ipv4_only] %>"
+LOGSTASH_OPTS="agent -f $LOGSTASH_HOME/etc/conf.d"
+# This is not compatible with logstash 1.5.0
+# LOGSTASH_OPTS="$LOGSTASH_OPTS --pluginpath $LOGSTASH_HOME/lib"
+<% if @options[:debug] -%>
+LOGSTASH_OPTS="$LOGSTASH_OPTS -vv"
+<% end -%>
+LOGSTASH_OPTS="$LOGSTASH_OPTS -l $LOGSTASH_HOME/log/<%= @options[:log_file] %>"
+export LOGSTASH_OPTS="$LOGSTASH_OPTS -w <%= @options[:workers] %>"
+<% if @options[:supervisor_gid] -%>
+HOME=$LOGSTASH_HOME exec chpst -u <%= @options[:user] %>:<%= @options[:supervisor_gid] %> $LOGSTASH_HOME/bin/logstash $LOGSTASH_OPTS
+<% else -%>
+HOME=$LOGSTASH_HOME exec chpst -u <%= @options[:user] %> $LOGSTASH_HOME/bin/logstash $LOGSTASH_OPTS
+<% end -%>