rubygems · May 12, 2015 · May 12, 2015 · May 12, 2015 · May 12, 2015 · May 12, 2015
diff --git a/.chef/bootstrap/rubygems-trusty.erb b/.chef/bootstrap/rubygems-trusty.erb
@@ -5,15 +5,6 @@ hostname -F /etc/hostname
 
 echo "`ip addr | grep eth0 | grep "inet " | cut -d " " -f 6 | cut -d \/ -f 1` `cat /etc/hostname`" >> /etc/hosts
 
-# fetch the rubygems package signing public key and add it to the apt keyring
-curl http://repo01.common.rubygems.org/packages.key | sudo apt-key add -
-
-# remove the AWS repos
-rm -f /etc/apt/sources.list
-
-# add the internal apt repo
-sh -c "echo \"deb     http://repo01.common.rubygems.org trusty main\" > /etc/apt/sources.list.d/rubygems_main.list"
-
 apt-get update
 apt-get upgrade -y
 

diff --git a/Berksfile.lock b/Berksfile.lock
@@ -299,7 +299,7 @@ GRAPH
     rsyslog (>= 0.0.0)
     rubygems-ruby (>= 0.0.0)
     runit (>= 0.0.0)
-  rubygems-logging (0.0.68)
+  rubygems-logging (0.0.69)
     chef-vault (>= 0.0.0)
     elasticsearch (>= 0.0.0)
     kibana_lwrp (>= 0.0.0)

diff --git a/Rakefile b/Rakefile
@@ -23,9 +23,11 @@ end
 desc 'Refresh all chef vaults'
 task :refresh_vaults do
   [
+    'apps/kibana',
     'aws/credentials',
     'certs/production',
     'certs/staging',
+    'certs/logstash',
     'datadog/credentials',
     'dnsimple/credentials',
     'duo/credentials',

diff --git a/cookbooks/rubygems-logging/metadata.rb b/cookbooks/rubygems-logging/metadata.rb
@@ -1,10 +1,11 @@
 name 'rubygems-logging'
 maintainer 'RubyGems.org ops team'
 
-version '0.0.68'
+version '0.0.69'
 
 depends 'chef-vault'
 depends 'elasticsearch'
+depends 'java'
 depends 'kibana_lwrp'
 depends 'logstash'
 depends 'nginx'

diff --git a/cookbooks/rubygems-logging/recipes/client.rb b/cookbooks/rubygems-logging/recipes/client.rb
@@ -89,6 +89,6 @@
 end
 
 service 'logstash-forwarder' do
-  supports :status => true, :restart => true, :reload => true
+  supports status: true, restart: true, reload: true
   action config['files'].empty? ? [:disable, :stop] : [:enable, :start]
 end
diff --git a/cookbooks/rubygems-logging/recipes/server_nginx.rb b/cookbooks/rubygems-logging/recipes/server_nginx.rb
@@ -32,6 +32,15 @@
   notifies :reload, 'service[nginx]'
 end
 
+auth_bag = chef_vault_item('apps', 'kibana')
+
+file "#{node['nginx']['dir']}/htpasswd" do
+  owner 'root'
+  group 'root'
+  mode '0644'
+  content "#{auth_bag['user']}:#{auth_bag['htpasswd']}"
+end
+
 template "#{node['nginx']['dir']}/sites-available/kibana" do
   source 'nginx.conf.erb'
   owner  'root'

diff --git a/cookbooks/rubygems-logging/templates/default/logstash/filters.conf.erb b/cookbooks/rubygems-logging/templates/default/logstash/filters.conf.erb
@@ -4,6 +4,7 @@ filter {
       rename => [ 'logsource', 'host' ]
     }
   }
+
   if [type] == 'rails' {
     grok {
       match => [
@@ -12,6 +13,7 @@ filter {
       ]
     }
   }
+
   if [type] == 'nginx_json' {
     json {
       source => 'message'
@@ -20,4 +22,60 @@ filter {
       source => 'client_ip'
     }
   }
+
+  if [type] == 'nginx_error' {
+    grok {
+      match => {
+        "message" => [
+          "%{DATESTAMP:mydate} \[%{DATA:severity_label}\] (%{DATA:pid}#%{NUMBER}: \*%{NUMBER}|\*%{NUMBER}) %{GREEDYDATA:mymessage}",
+          "%{DATESTAMP:mydate} \[%{DATA:severity_label}\] %{GREEDYDATA:mymessage}",
+          "%{DATESTAMP:mydate} %{GREEDYDATA:mymessage}"
+        ]
+      }
+    }
+    date {
+      match => [ "mydate", "yy/MM/dd HH:mm:ss" ]
+      remove_field => ["mydate"]
+    }
+    grok {
+      match => {
+        "mymessage" => [
+          "host: \"%{IPORHOST:[dest_host]}\""
+        ]
+      }
+      add_tag => ["add_nginx_error_dest_host"]
+    }
+    grok {
+      match => {
+        "mymessage" => [
+          "request: \"%{WORD:[method]} %{DATA:[uri]} HTTP/%{NUMBER:[http_version]:float}\""
+        ]
+      }
+      add_tag => ["add_nginx_error_request"]
+    }
+    grok {
+      match => {
+        "mymessage" => [
+          "client: %{IPORHOST:[client_ip]}",
+          "client %{IP:[client_ip]} "
+        ]
+      }
+      add_tag => ["add_nginx_error_client_ip"]
+    }
+    grok {
+      match => {
+        "mymessage" => [
+          "referrer: \"%{DATA:[request_referrer]}\""
+        ]
+      }
+      add_tag => ["add_nginx_error_referrer"]
+    }
+    geoip {
+      source => 'client_ip'
+    }
+    mutate {
+      rename => [ "message", "original_message"]
+      rename => [ "mymessage", "message" ]
+    }
+  }
 }
diff --git a/cookbooks/rubygems-logging/templates/default/nginx.conf.erb b/cookbooks/rubygems-logging/templates/default/nginx.conf.erb
@@ -18,6 +18,9 @@ server {
   proxy_set_header Host $host;
   proxy_redirect off;
 
+  auth_basic "logs.rubygems.org";
+  auth_basic_user_file /etc/nginx/htpasswd;
+
   location / {
     proxy_pass http://localhost:5601;
   }