Skip to content
This repository has been archived by the owner on Jul 11, 2020. It is now read-only.

Commit

Permalink
hook up PagerDuty
Browse files Browse the repository at this point in the history
dwradcliffe committed Jul 25, 2014

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent e8b36ac commit 0953bf3
Showing 11 changed files with 108 additions and 16 deletions.
2 changes: 1 addition & 1 deletion Berksfile.lock
Original file line number Diff line number Diff line change
@@ -277,7 +277,7 @@ GRAPH
nginx (>= 0.0.0)
rubygems (>= 0.0.0)
rubygems-ruby (0.0.1)
rubygems-sensu (0.0.68)
rubygems-sensu (0.0.71)
chef-vault (>= 0.0.0)
sensu (>= 0.0.0)
rubygems-ssh (0.0.4)
51 changes: 51 additions & 0 deletions cookbooks/rubygems-sensu/files/default/pagerduty.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
#!/usr/bin/env ruby
#
# This handler creates and resolves PagerDuty incidents, refreshing
# stale incident details every 30 minutes
#
# Copyright 2011 Sonian, Inc <chefs@sonian.net>
#
# Released under the same terms as Sensu (the MIT license); see LICENSE
# for details.

require 'rubygems' if RUBY_VERSION < '1.9.0'
require 'sensu-handler'
require 'redphone/pagerduty'

class Pagerduty < Sensu::Handler

def incident_key
@event['client']['name'] + '/' + @event['check']['name']
end

def handle
description = @event['check']['notification']
description ||= [@event['client']['name'], @event['check']['name'], @event['check']['output']].join(' : ')
begin
timeout(10) do
response = case @event['action']
when 'create'
Redphone::Pagerduty.trigger_incident(
:service_key => settings['pagerduty']['api_key'],
:incident_key => incident_key,
:description => description,
:details => @event
)
when 'resolve'
Redphone::Pagerduty.resolve_incident(
:service_key => settings['pagerduty']['api_key'],
:incident_key => incident_key
)
end
if response['status'] == 'success'
puts 'pagerduty -- ' + @event['action'].capitalize + 'd incident -- ' + incident_key
else
puts 'pagerduty -- failed to ' + @event['action'] + ' incident -- ' + incident_key
end
end
rescue Timeout::Error
puts 'pagerduty -- timed out while attempting to ' + @event['action'] + ' a incident -- ' + incident_key
end
end

end
2 changes: 1 addition & 1 deletion cookbooks/rubygems-sensu/metadata.rb
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
name 'rubygems-sensu'
maintainer 'RubyGems.org ops team'

version '0.0.68'
version '0.0.71'

depends 'chef-vault'
depends 'sensu'
6 changes: 3 additions & 3 deletions cookbooks/rubygems-sensu/recipes/app.rb
Original file line number Diff line number Diff line change
@@ -5,23 +5,23 @@

sensu_check 'check_app_nginx_http' do
command '/usr/lib/nagios/plugins/check_http -I 127.0.0.1 -p 9000 -t 5 -w 1 -c 2'
handlers ['slack']
handlers ['slack', 'pagerduty']
subscribers ['app']
interval 30
additional(notification: 'nginx is not returning 200 OK on port 9000', occurences: 3)
end

sensu_check 'check_unicorn_http' do
command '/usr/lib/nagios/plugins/check_http -I 127.0.0.1 -p 3000 -t 5 -w 1 -c 2'
handlers ['slack']
handlers ['slack', 'pagerduty']
subscribers ['app']
interval 30
additional(notification: 'unicorn is not returning 200 OK on port 3000', occurences: 3)
end

sensu_check 'check_unicorn_proc' do
command "/opt/sensu/embedded/bin/ruby /etc/sensu/plugins/check-procs.rb -p 'unicorn master'"
handlers ['slack']
handlers ['slack', 'pagerduty']
subscribers ['app']
interval 30
additional(notification: 'unicorn is not running', occurences: 3)
6 changes: 3 additions & 3 deletions cookbooks/rubygems-sensu/recipes/balancer.rb
Original file line number Diff line number Diff line change
@@ -5,23 +5,23 @@

sensu_check 'check_nginx_http' do
command '/usr/lib/nagios/plugins/check_http -I 127.0.0.1 -t 5 -w 1 -c 2'
handlers ['slack']
handlers ['slack', 'pagerduty']
subscribers ['balancer']
interval 30
additional(notification: 'nginx is not returning 200 OK on port 80', occurences: 3)
end

sensu_check 'check_nginx_https' do
command '/usr/lib/nagios/plugins/check_http -I 127.0.0.1 --ssl -t 5 -w 1 -c 2'
handlers ['slack']
handlers ['slack', 'pagerduty']
subscribers ['balancer']
interval 30
additional(notification: 'nginx is not returning 200 OK on port 443', occurences: 3)
end

sensu_check 'check_nginx_cert_expiration' do
command '/usr/lib/nagios/plugins/check_http -I 127.0.0.1 --ssl -C 30,14'
handlers ['slack']
handlers ['slack', 'pagerduty']
subscribers ['balancer']
interval 30
additional(occurences: 3)
14 changes: 7 additions & 7 deletions cookbooks/rubygems-sensu/recipes/base.rb
Original file line number Diff line number Diff line change
@@ -22,55 +22,55 @@

sensu_check 'check_procs' do
command '/opt/sensu/embedded/bin/ruby /etc/sensu/plugins/check-procs.rb'
handlers ['slack']
handlers ['slack', 'pagerduty']
subscribers ['all']
interval 30
additional(notification: 'There is a high number of procs running', occurences: 3)
end

sensu_check 'check_ntpd_proc' do
command "/opt/sensu/embedded/bin/ruby /etc/sensu/plugins/check-procs.rb -p '/usr/sbin/ntpd -p /var/run/ntpd.pid'"
handlers ['slack']
handlers ['slack', 'pagerduty']
subscribers ['all']
interval 30
additional(notification: 'ntpd is not running', occurences: 3)
end

sensu_check 'check_collectd_proc' do
command "/opt/sensu/embedded/bin/ruby /etc/sensu/plugins/check-procs.rb -p 'collectd -C /etc/collectd/collectd.conf'"
handlers ['slack']
handlers ['slack', 'pagerduty']
subscribers ['all']
interval 30
additional(notification: 'collectd is not running', occurences: 3)
end

sensu_check 'check_ssh' do
command '/usr/lib/nagios/plugins/check_ssh localhost'
handlers ['slack']
handlers ['slack', 'pagerduty']
subscribers ['all']
interval 30
additional(notification: 'sshd is not running', occurences: 3)
end

sensu_check 'check_apt' do
command '/usr/lib/nagios/plugins/check_apt'
handlers ['slack']
handlers ['slack', 'pagerduty']
subscribers ['all']
interval 60
additional(notification: 'There are pending package upgrades', occurences: 3)
end

sensu_check 'check_ntp_time' do
command '/usr/lib/nagios/plugins/check_ntp_time -H localhost'
handlers ['slack']
handlers ['slack', 'pagerduty']
subscribers ['all']
interval 120
additional(notification: 'NTP is out of sync', occurences: 3)
end

sensu_check 'check_load' do
command "/usr/lib/nagios/plugins/check_load -w #{node['cpu']['total'] * 8}:#{node['cpu']['total'] * 5}:#{node['cpu']['total'] * 2} -c #{node['cpu']['total'] * 10}:#{node['cpu']['total'] * 8}:#{node['cpu']['total'] * 3}"
handlers ['slack']
handlers ['slack', 'pagerduty']
subscribers ['all']
interval 30
additional(notification: 'Load is high', occurences: 3)
2 changes: 1 addition & 1 deletion cookbooks/rubygems-sensu/recipes/nginx.rb
Original file line number Diff line number Diff line change
@@ -5,7 +5,7 @@

sensu_check 'check_nginx_proc' do
command "/opt/sensu/embedded/bin/ruby /etc/sensu/plugins/check-procs.rb -p 'nginx: master process /usr/sbin/nginx'"
handlers ['slack']
handlers ['slack', 'pagerduty']
subscribers ['balancer', 'app']
interval 30
additional(notification: 'nginx is not running', occurences: 3)
34 changes: 34 additions & 0 deletions cookbooks/rubygems-sensu/recipes/pagerduty.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
#
# Cookbook Name:: rubygems-sensu
# Recipe:: pagerduty
#

include_recipe 'chef-vault'

pagerduty_creds = chef_vault_item('sensu', 'credentials')['pagerduty']

template '/etc/sensu/conf.d/pagerduty.json' do
source 'pagerduty.json.erb'
owner 'sensu'
group 'sensu'
variables(
api_key: pagerduty_creds['api_key'],
)
end

gem_package 'redphone' do
gem_binary '/opt/sensu/embedded/bin/gem'
end

cookbook_file '/etc/sensu/handlers/pagerduty.rb' do
path '/etc/sensu/handlers/pagerduty.rb'
source 'pagerduty.rb'
mode "0755"
action :create
end

sensu_handler 'pagerduty' do
type 'pipe'
command 'pagerduty.rb'
severities ['ok', 'critical']
end
1 change: 1 addition & 0 deletions cookbooks/rubygems-sensu/recipes/server.rb
Original file line number Diff line number Diff line change
@@ -20,6 +20,7 @@

include_recipe 'rubygems-sensu::librato'
include_recipe 'rubygems-sensu::slack'
include_recipe 'rubygems-sensu::pagerduty'

include_recipe 'sensu::rabbitmq'

1 change: 1 addition & 0 deletions cookbooks/rubygems-sensu/recipes/slack.rb
Original file line number Diff line number Diff line change
@@ -22,6 +22,7 @@
cookbook_file '/etc/sensu/handlers/slack.rb' do
path '/etc/sensu/handlers/slack.rb'
source 'slack.rb'
mode "0755"
action :create
end

5 changes: 5 additions & 0 deletions cookbooks/rubygems-sensu/templates/default/pagerduty.json.erb
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"pagerduty": {
"api_key": "<%= @api_key %>"
}
}

0 comments on commit 0953bf3

Please sign in to comment.