hook up PagerDuty

rubygems · Jul 25, 2014 · 0953bf3 · 0953bf3
1 parent e8b36ac
commit 0953bf3
Show file tree

Hide file tree

Showing 11 changed files with 108 additions and 16 deletions.
diff --git a/Berksfile.lock b/Berksfile.lock
@@ -277,7 +277,7 @@ GRAPH
     nginx (>= 0.0.0)
     rubygems (>= 0.0.0)
   rubygems-ruby (0.0.1)
-  rubygems-sensu (0.0.68)
+  rubygems-sensu (0.0.71)
     chef-vault (>= 0.0.0)
     sensu (>= 0.0.0)
   rubygems-ssh (0.0.4)

diff --git a/cookbooks/rubygems-sensu/files/default/pagerduty.rb b/cookbooks/rubygems-sensu/files/default/pagerduty.rb
@@ -0,0 +1,51 @@
+#!/usr/bin/env ruby
+#
+# This handler creates and resolves PagerDuty incidents, refreshing
+# stale incident details every 30 minutes
+#
+# Copyright 2011 Sonian, Inc <chefs@sonian.net>
+#
+# Released under the same terms as Sensu (the MIT license); see LICENSE
+# for details.
+
+require 'rubygems' if RUBY_VERSION < '1.9.0'
+require 'sensu-handler'
+require 'redphone/pagerduty'
+
+class Pagerduty < Sensu::Handler
+
+  def incident_key
+    @event['client']['name'] + '/' + @event['check']['name']
+  end
+
+  def handle
+    description = @event['check']['notification']
+    description ||= [@event['client']['name'], @event['check']['name'], @event['check']['output']].join(' : ')
+    begin
+      timeout(10) do
+        response = case @event['action']
+        when 'create'
+          Redphone::Pagerduty.trigger_incident(
+            :service_key => settings['pagerduty']['api_key'],
+            :incident_key => incident_key,
+            :description => description,
+            :details => @event
+          )
+        when 'resolve'
+          Redphone::Pagerduty.resolve_incident(
+            :service_key => settings['pagerduty']['api_key'],
+            :incident_key => incident_key
+          )
+        end
+        if response['status'] == 'success'
+          puts 'pagerduty -- ' + @event['action'].capitalize + 'd incident -- ' + incident_key
+        else
+          puts 'pagerduty -- failed to ' + @event['action'] + ' incident -- ' + incident_key
+        end
+      end
+    rescue Timeout::Error
+      puts 'pagerduty -- timed out while attempting to ' + @event['action'] + ' a incident -- ' + incident_key
+    end
+  end
+
+end
diff --git a/cookbooks/rubygems-sensu/metadata.rb b/cookbooks/rubygems-sensu/metadata.rb
@@ -1,7 +1,7 @@
 name 'rubygems-sensu'
 maintainer 'RubyGems.org ops team'
 
-version '0.0.68'
+version '0.0.71'
 
 depends 'chef-vault'
 depends 'sensu'

diff --git a/cookbooks/rubygems-sensu/recipes/app.rb b/cookbooks/rubygems-sensu/recipes/app.rb
@@ -5,23 +5,23 @@
 
 sensu_check 'check_app_nginx_http' do
   command '/usr/lib/nagios/plugins/check_http -I 127.0.0.1 -p 9000 -t 5 -w 1 -c 2'
-  handlers ['slack']
+  handlers ['slack', 'pagerduty']
   subscribers ['app']
   interval 30
   additional(notification: 'nginx is not returning 200 OK on port 9000', occurences: 3)
 end
 
 sensu_check 'check_unicorn_http' do
   command '/usr/lib/nagios/plugins/check_http -I 127.0.0.1 -p 3000 -t 5 -w 1 -c 2'
-  handlers ['slack']
+  handlers ['slack', 'pagerduty']
   subscribers ['app']
   interval 30
   additional(notification: 'unicorn is not returning 200 OK on port 3000', occurences: 3)
 end
 
 sensu_check 'check_unicorn_proc' do
   command "/opt/sensu/embedded/bin/ruby /etc/sensu/plugins/check-procs.rb -p 'unicorn master'"
-  handlers ['slack']
+  handlers ['slack', 'pagerduty']
   subscribers ['app']
   interval 30
   additional(notification: 'unicorn is not running', occurences: 3)

diff --git a/cookbooks/rubygems-sensu/recipes/balancer.rb b/cookbooks/rubygems-sensu/recipes/balancer.rb
@@ -5,23 +5,23 @@
 
 sensu_check 'check_nginx_http' do
   command '/usr/lib/nagios/plugins/check_http -I 127.0.0.1 -t 5 -w 1 -c 2'
-  handlers ['slack']
+  handlers ['slack', 'pagerduty']
   subscribers ['balancer']
   interval 30
   additional(notification: 'nginx is not returning 200 OK on port 80', occurences: 3)
 end
 
 sensu_check 'check_nginx_https' do
   command '/usr/lib/nagios/plugins/check_http -I 127.0.0.1 --ssl -t 5 -w 1 -c 2'
-  handlers ['slack']
+  handlers ['slack', 'pagerduty']
   subscribers ['balancer']
   interval 30
   additional(notification: 'nginx is not returning 200 OK on port 443', occurences: 3)
 end
 
 sensu_check 'check_nginx_cert_expiration' do
   command '/usr/lib/nagios/plugins/check_http -I 127.0.0.1 --ssl -C 30,14'
-  handlers ['slack']
+  handlers ['slack', 'pagerduty']
   subscribers ['balancer']
   interval 30
   additional(occurences: 3)

diff --git a/cookbooks/rubygems-sensu/recipes/base.rb b/cookbooks/rubygems-sensu/recipes/base.rb
@@ -22,55 +22,55 @@
 
 sensu_check 'check_procs' do
   command '/opt/sensu/embedded/bin/ruby /etc/sensu/plugins/check-procs.rb'
-  handlers ['slack']
+  handlers ['slack', 'pagerduty']
   subscribers ['all']
   interval 30
   additional(notification: 'There is a high number of procs running', occurences: 3)
 end
 
 sensu_check 'check_ntpd_proc' do
   command "/opt/sensu/embedded/bin/ruby /etc/sensu/plugins/check-procs.rb -p '/usr/sbin/ntpd -p /var/run/ntpd.pid'"
-  handlers ['slack']
+  handlers ['slack', 'pagerduty']
   subscribers ['all']
   interval 30
   additional(notification: 'ntpd is not running', occurences: 3)
 end
 
 sensu_check 'check_collectd_proc' do
   command "/opt/sensu/embedded/bin/ruby /etc/sensu/plugins/check-procs.rb -p 'collectd -C /etc/collectd/collectd.conf'"
-  handlers ['slack']
+  handlers ['slack', 'pagerduty']
   subscribers ['all']
   interval 30
   additional(notification: 'collectd is not running', occurences: 3)
 end
 
 sensu_check 'check_ssh' do
   command '/usr/lib/nagios/plugins/check_ssh localhost'
-  handlers ['slack']
+  handlers ['slack', 'pagerduty']
   subscribers ['all']
   interval 30
   additional(notification: 'sshd is not running', occurences: 3)
 end
 
 sensu_check 'check_apt' do
   command '/usr/lib/nagios/plugins/check_apt'
-  handlers ['slack']
+  handlers ['slack', 'pagerduty']
   subscribers ['all']
   interval 60
   additional(notification: 'There are pending package upgrades', occurences: 3)
 end
 
 sensu_check 'check_ntp_time' do
   command '/usr/lib/nagios/plugins/check_ntp_time -H localhost'
-  handlers ['slack']
+  handlers ['slack', 'pagerduty']
   subscribers ['all']
   interval 120
   additional(notification: 'NTP is out of sync', occurences: 3)
 end
 
 sensu_check 'check_load' do
   command "/usr/lib/nagios/plugins/check_load -w #{node['cpu']['total'] * 8}:#{node['cpu']['total'] * 5}:#{node['cpu']['total'] * 2} -c #{node['cpu']['total'] * 10}:#{node['cpu']['total'] * 8}:#{node['cpu']['total'] * 3}"
-  handlers ['slack']
+  handlers ['slack', 'pagerduty']
   subscribers ['all']
   interval 30
   additional(notification: 'Load is high', occurences: 3)

diff --git a/cookbooks/rubygems-sensu/recipes/nginx.rb b/cookbooks/rubygems-sensu/recipes/nginx.rb
@@ -5,7 +5,7 @@
 
 sensu_check 'check_nginx_proc' do
   command "/opt/sensu/embedded/bin/ruby /etc/sensu/plugins/check-procs.rb -p 'nginx: master process /usr/sbin/nginx'"
-  handlers ['slack']
+  handlers ['slack', 'pagerduty']
   subscribers ['balancer', 'app']
   interval 30
   additional(notification: 'nginx is not running', occurences: 3)

diff --git a/cookbooks/rubygems-sensu/recipes/pagerduty.rb b/cookbooks/rubygems-sensu/recipes/pagerduty.rb
@@ -0,0 +1,34 @@
+#
+# Cookbook Name:: rubygems-sensu
+# Recipe:: pagerduty
+#
+
+include_recipe 'chef-vault'
+
+pagerduty_creds = chef_vault_item('sensu', 'credentials')['pagerduty']
+
+template '/etc/sensu/conf.d/pagerduty.json' do
+  source 'pagerduty.json.erb'
+  owner 'sensu'
+  group 'sensu'
+  variables(
+    api_key: pagerduty_creds['api_key'],
+  )
+end
+
+gem_package 'redphone' do
+  gem_binary '/opt/sensu/embedded/bin/gem'
+end
+
+cookbook_file '/etc/sensu/handlers/pagerduty.rb' do
+  path '/etc/sensu/handlers/pagerduty.rb'
+  source 'pagerduty.rb'
+  mode "0755"
+  action :create
+end
+
+sensu_handler 'pagerduty' do
+  type 'pipe'
+  command 'pagerduty.rb'
+  severities ['ok', 'critical']
+end
diff --git a/cookbooks/rubygems-sensu/recipes/server.rb b/cookbooks/rubygems-sensu/recipes/server.rb
@@ -20,6 +20,7 @@
 
 include_recipe 'rubygems-sensu::librato'
 include_recipe 'rubygems-sensu::slack'
+include_recipe 'rubygems-sensu::pagerduty'
 
 include_recipe 'sensu::rabbitmq'
 

diff --git a/cookbooks/rubygems-sensu/recipes/slack.rb b/cookbooks/rubygems-sensu/recipes/slack.rb
@@ -22,6 +22,7 @@
 cookbook_file '/etc/sensu/handlers/slack.rb' do
   path '/etc/sensu/handlers/slack.rb'
   source 'slack.rb'
+  mode "0755"
   action :create
 end
 

diff --git a/cookbooks/rubygems-sensu/templates/default/pagerduty.json.erb b/cookbooks/rubygems-sensu/templates/default/pagerduty.json.erb
@@ -0,0 +1,5 @@
+{
+  "pagerduty": {
+    "api_key": "<%= @api_key %>"
+  }
+}