rubygems · Jun 20, 2015 · Jun 20, 2015
Showing with 44 additions and 4 deletions.

+2 −3 Berksfile.lock

+1 −1 cookbooks/rubygems-logging/metadata.rb

+41 −0 cookbooks/rubygems-logging/recipes/server_elasticsearch.rb
diff --git a/Berksfile.lock b/Berksfile.lock
@@ -211,8 +211,7 @@ GRAPH
     yum (>= 0.0.0)
   rabbitmq (3.3.0)
     erlang (>= 0.9)
-  redisio (2.2.4)
-    build-essential (>= 0.0.0)
+  redisio (1.7.1)
     ulimit (>= 0.1.2)
   route53 (0.3.8)
     build-essential (>= 0.0.0)
@@ -304,7 +303,7 @@ GRAPH
     rsyslog (>= 0.0.0)
     rubygems-ruby (>= 0.0.0)
     runit (>= 0.0.0)
-  rubygems-logging (0.0.69)
+  rubygems-logging (0.0.75)
     chef-vault (>= 0.0.0)
     elasticsearch (>= 0.0.0)
     java (>= 0.0.0)

diff --git a/cookbooks/rubygems-logging/metadata.rb b/cookbooks/rubygems-logging/metadata.rb
@@ -1,7 +1,7 @@
 name 'rubygems-logging'
 maintainer 'RubyGems.org ops team'
 
-version '0.0.69'
+version '0.0.75'
 
 depends 'chef-vault'
 depends 'elasticsearch'

diff --git a/cookbooks/rubygems-logging/recipes/server_elasticsearch.rb b/cookbooks/rubygems-logging/recipes/server_elasticsearch.rb
@@ -9,6 +9,37 @@
 node.default['elasticsearch']['cloud']['aws']['region'] = 'us-west-2'
 node.default['elasticsearch']['version'] = '1.5.0'
 node.default['elasticsearch']['allocated_memory'] = '4000m'
+node.default['elasticsearch']['bootstrap']['mlockall'] = true
+
+# node.default['elasticsearch']['custom_config']['threadpool.search.type'] = 'fixed'
+# node.default['elasticsearch']['custom_config']['threadpool.search.size'] = '20'
+# node.default['elasticsearch']['custom_config']['threadpool.search.queue_size'] = '100'
+
+node.default['elasticsearch']['custom_config']['threadpool.bulk.type'] = 'fixed'
+node.default['elasticsearch']['custom_config']['threadpool.bulk.size'] = '60'
+node.default['elasticsearch']['custom_config']['threadpool.bulk.queue_size'] = '300'
+
+node.default['elasticsearch']['custom_config']['threadpool.index.type'] = 'fixed'
+node.default['elasticsearch']['custom_config']['threadpool.index.size'] = '20'
+node.default['elasticsearch']['custom_config']['threadpool.index.queue_size'] = '100'
+
+node.default['elasticsearch']['custom_config']['indices.memory.index_buffer_size'] = '30%'
+node.default['elasticsearch']['custom_config']['indices.memory.min_shard_index_buffer_size'] = '12mb'
+node.default['elasticsearch']['custom_config']['indices.memory.min_index_buffer_size'] = '96mb'
+
+node.default['elasticsearch']['custom_config']['indices.fielddata.cache.size'] = '15%'
+node.default['elasticsearch']['custom_config']['indices.fielddata.cache.expire'] = '6h'
+node.default['elasticsearch']['custom_config']['indices.cache.filter.size'] = '15%'
+node.default['elasticsearch']['custom_config']['indices.cache.filter.expire'] = '6h'
+
+node.default['elasticsearch']['custom_config']['index.refresh_interval'] = '30s'
+node.default['elasticsearch']['custom_config']['index.translog.flush_threshold_ops'] = '50000'
+
+node.default['elasticsearch']['gc_settings'] =<<-CONFIG
+  -XX:+UseG1GC
+  -XX:+HeapDumpOnOutOfMemoryError
+CONFIG
+
 node.default['elasticsearch']['path']['data'] = ['/data/elasticsearch/disk1', '/data/elasticsearch/disk2']
 node.default['elasticsearch']['data']['devices'] = {
   '/dev/xvdf' => {
@@ -46,3 +77,13 @@
 include_recipe 'elasticsearch::data'
 
 install_plugin 'lmenezes/elasticsearch-kopf'
+
+easy_install_package 'elasticsearch-curator'
+
+cron_d 'delete_old_logstash_indexes' do
+  user 'root'
+  action :create
+  hour '9'
+  minute '0'
+  command "/usr/local/bin/curator delete indices --older-than 30 --time-unit days --prefix logstash- --timestring '\\%Y.\\%m.\\%d'"
+end