add shipit

rubygems · May 22, 2015 · 6e9c334 · 6e9c334
1 parent d5d22ae
commit 6e9c334
Showing 22 changed files with 384 additions and 94 deletions.
diff --git a/Berksfile b/Berksfile
@@ -28,6 +28,7 @@ cookbook 'rubygems-redis', path: 'cookbooks/rubygems-redis'
 cookbook 'rubygems-repo', path: 'cookbooks/rubygems-repo'
 cookbook 'rubygems-ruby', path: 'cookbooks/rubygems-ruby'
 cookbook 'rubygems-sensu', path: 'cookbooks/rubygems-sensu'
+cookbook 'rubygems-shipit', path: 'cookbooks/rubygems-shipit'
 cookbook 'rubygems-ssh', path: 'cookbooks/rubygems-ssh'
 cookbook 'rubygems-stat-update', path: 'cookbooks/rubygems-stat-update'
 cookbook 'rubygems-utility', path: 'cookbooks/rubygems-utility'

diff --git a/Berksfile.lock b/Berksfile.lock
@@ -58,6 +58,8 @@ DEPENDENCIES
     path: cookbooks/rubygems-ruby
   rubygems-sensu
     path: cookbooks/rubygems-sensu
+  rubygems-shipit
+    path: cookbooks/rubygems-shipit
   rubygems-ssh
     path: cookbooks/rubygems-ssh
   rubygems-stat-update
@@ -232,7 +234,7 @@ GRAPH
     rubygems-sensu (>= 0.0.0)
     rubygems-ssh (>= 0.0.0)
     rubygems-utility (>= 0.0.0)
-  rubygems-app (0.0.49)
+  rubygems-app (0.0.52)
     apt (>= 0.0.0)
     chef-vault (>= 0.0.0)
     datadog (>= 0.0.0)
@@ -243,6 +245,7 @@ GRAPH
     rubygems (>= 0.0.0)
     rubygems-ruby (>= 0.0.0)
     runit (>= 0.0.0)
+    sudo (>= 0.0.0)
     unicorn (>= 0.0.0)
     user (>= 0.0.0)
   rubygems-apt (0.0.19)
@@ -291,7 +294,7 @@ GRAPH
     dns (>= 0.0.0)
     hostname (>= 0.0.0)
   rubygems-hosts (0.0.10)
-  rubygems-hubot (0.0.68)
+  rubygems-hubot (0.0.69)
     chef-vault (>= 0.0.0)
     hubot (~> 1.0.2)
     nginx (>= 0.0.0)
@@ -302,6 +305,7 @@ GRAPH
   rubygems-logging (0.0.69)
     chef-vault (>= 0.0.0)
     elasticsearch (>= 0.0.0)
+    java (>= 0.0.0)
     kibana_lwrp (>= 0.0.0)
     logstash (>= 0.0.0)
     nginx (>= 0.0.0)
@@ -310,7 +314,7 @@ GRAPH
     motd (>= 0.0.0)
   rubygems-ntp (0.0.3)
     ntp (>= 0.0.0)
-  rubygems-people (0.0.36)
+  rubygems-people (0.0.38)
     sudo (>= 0.0.0)
     user (>= 0.0.0)
   rubygems-redis (0.1.8)
@@ -334,6 +338,13 @@ GRAPH
     rubygems (>= 0.0.0)
     sensu (= 2.1.0)
     uchiwa (= 1.0.0)
+  rubygems-shipit (0.0.44)
+    apt (>= 0.0.0)
+    git (>= 0.0.0)
+    logrotate (>= 0.0.0)
+    nodejs (>= 0.0.0)
+    runit (>= 0.0.0)
+    sudo (>= 0.0.0)
   rubygems-ssh (0.0.5)
     openssh (>= 0.0.0)
   rubygems-stat-update (0.0.1)

diff --git a/Rakefile b/Rakefile
@@ -24,6 +24,7 @@ desc 'Refresh all chef vaults'
 task :refresh_vaults do
   [
     'apps/kibana',
+    'apps/shipit',
     'aws/credentials',
     'certs/production',
     'certs/staging',

diff --git a/cookbooks/rubygems-app/metadata.rb b/cookbooks/rubygems-app/metadata.rb
@@ -1,6 +1,6 @@
 name 'rubygems-app'
 
-version '0.0.49'
+version '0.0.52'
 
 depends 'apt'
 depends 'chef-vault'
@@ -12,5 +12,6 @@
 depends 'rubygems'
 depends 'rubygems-ruby'
 depends 'runit'
+depends 'sudo'
 depends 'unicorn'
 depends 'user'
diff --git a/cookbooks/rubygems-app/recipes/default.rb b/cookbooks/rubygems-app/recipes/default.rb
@@ -17,7 +17,7 @@
 
 package 'libpq-dev'
 
-include_recipe 'rubygems-app::deploy_user'
+include_recipe 'rubygems-people::deploy'
 include_recipe 'rubygems-app::dirs'
 include_recipe 'rubygems-app::config'
 include_recipe 'rubygems-app::unicorn'

diff --git a/cookbooks/rubygems-app/recipes/delayed_job.rb b/cookbooks/rubygems-app/recipes/delayed_job.rb
@@ -23,6 +23,15 @@
   action ::File.exist?('/applications/rubygems/current') ? :enable : :disable
 end
 
+sudo 'deploy-delayed_job' do
+  user 'deploy'
+  commands [
+    '/etc/init.d/delayed_job *',
+    '/usr/sbin/service delayed_job *'
+  ]
+  nopasswd true
+end
+
 template '/usr/local/bin/background_job_stats.sh' do
   source 'background_job_stats.sh.erb'
   mode '0755'

diff --git a/cookbooks/rubygems-app/recipes/unicorn.rb b/cookbooks/rubygems-app/recipes/unicorn.rb
@@ -33,3 +33,12 @@
   )
   action ::File.exist?('/applications/rubygems/current') ? :enable : :disable
 end
+
+sudo 'deploy-unicorn' do
+  user 'deploy'
+  commands [
+    '/etc/init.d/unicorn *',
+    '/usr/sbin/service unicorn *'
+  ]
+  nopasswd true
+end
diff --git a/cookbooks/rubygems-hubot/files/default/scripts/deploy.coffee b/cookbooks/rubygems-hubot/files/default/scripts/deploy.coffee
@@ -13,90 +13,7 @@
 # Author:
 #   David Radcliffe
 
-child_process = require 'child_process'
-fs = require 'fs'
-async = require 'async'
-
-exec = (command, logFile, callback) ->
-  fs.appendFile(logFile, command + "\n")
-
-  options = { cwd: process.env["HUBOT_DEPLOY_DIR"], env: { HOME: process.env["HOME"], PATH: process.env["PATH"], PWD: process.env["HUBOT_DEPLOY_DIR"] } }
-  proc = child_process.exec(command, options)
-
-  proc.stdout.on "data", (chunk) ->
-    fs.appendFile(logFile, chunk)
-
-  proc.stderr.on "data", (chunk) ->
-    fs.appendFile(logFile, chunk)
-
-  proc.on "error", (err) ->
-    callback "Error: #{err}"
-
-  proc.on "close", (exit_code)->
-    if exit_code == 0
-      callback null
-    else
-      callback "Exited with #{exit_code}"
-
-allowedToDeploy = (username, env) ->
-  return username in ['dwradcliffe', 'qrush', 'evanphx', 'shk', 'arthurnn', 'sferik']
-
-deploy = (msg, env, branch) ->
-  unless allowedToDeploy(msg.message.user.name, env)
-    console.log "#{msg.message.user.name} tried to deploy!"
-    msg.send "You are not allowed to deploy! Sorry!"
-  else
-    deployKey = Math.round(+new Date()/1000)
-    url = "https://bot.rubygems.org/deploy/#{deployKey}"
-    if branch
-      msg.send "Deploying #{branch} branch to #{env}..."
-    else
-      msg.send "Deploying to #{env}..."
-
-    async.series [
-      (callback) ->
-        exec "git fetch origin", "#{process.env["HUBOT_DEPLOY_LOG_DIR"]}/#{deployKey}.log", callback
-      (callback) ->
-        if branch
-          exec "git checkout #{branch}", "#{process.env["HUBOT_DEPLOY_LOG_DIR"]}/#{deployKey}.log", callback
-        else
-          exec "git checkout master", "#{process.env["HUBOT_DEPLOY_LOG_DIR"]}/#{deployKey}.log", callback
-      (callback) ->
-        if branch
-          exec "git pull --force origin #{branch}", "#{process.env["HUBOT_DEPLOY_LOG_DIR"]}/#{deployKey}.log", callback
-        else
-          exec "git pull --force origin master", "#{process.env["HUBOT_DEPLOY_LOG_DIR"]}/#{deployKey}.log", callback
-      (callback) ->
-        exec "git submodule update --init --recursive", "#{process.env["HUBOT_DEPLOY_LOG_DIR"]}/#{deployKey}.log", callback
-      (callback) ->
-        localFlag = if env is 'staging' then '' else '--local'
-        exec "bundle check || bundle install #{localFlag} --without production", "#{process.env["HUBOT_DEPLOY_LOG_DIR"]}/#{deployKey}.log", callback
-      (callback) ->
-        branchCmd = if branch then "BRANCH=#{branch}" else ''
-        cleanCmd = if 'staging' == env then 'deploy:clean_git_cache' else ''
-        exec "bundle exec cap #{env} #{cleanCmd} deploy -s user=hubot #{branchCmd}", "#{process.env["HUBOT_DEPLOY_LOG_DIR"]}/#{deployKey}.log", callback
-    ], (err, results) ->
-      if err
-        console.log err
-        msg.send ":x: Deploy to #{env} failed! #{url}"
-      else
-        msg.send ":+1: Deployed to #{env}! #{url}"
-
 module.exports = (robot) ->
 
-  robot.respond /deploy$/i, (msg) ->
-    msg.send 'Not quite ready for use yet!'
-
-  robot.respond /deploy\s+(production|staging)$/i, (msg) ->
-    env = msg.match[1]
-    deploy(msg, env)
-
-  robot.respond /deploy\s(\S+)\sto\sstaging$/i, (msg) ->
-    env = 'staging'
-    branch = msg.match[1] || 'master'
-    deploy(msg, env, branch)
-
-  robot.router.get '/deploy/:key', (req, res) ->
-    fs.readFile "#{process.env["HUBOT_DEPLOY_LOG_DIR"]}/#{req.params.key}.log", (err, data) ->
-      res.set 'Content-Type', 'text/plain'
-      res.send data
+  robot.respond /deploy/i, (msg) ->
+    msg.send 'Please use https://shipit.rubygems.org to deploy.'
diff --git a/cookbooks/rubygems-hubot/metadata.rb b/cookbooks/rubygems-hubot/metadata.rb
@@ -1,6 +1,6 @@
 name 'rubygems-hubot'
 
-version '0.0.68'
+version '0.0.69'
 
 depends 'chef-vault'
 depends 'hubot', '~> 1.0.2'

diff --git a/cookbooks/rubygems-people/metadata.rb b/cookbooks/rubygems-people/metadata.rb
@@ -2,7 +2,7 @@
 maintainer       'RubyGems.org Ops Team'
 license          'MIT'
 description      'RubyGems user recipes'
-version          '0.0.36'
+version          '0.0.38'
 
 depends 'user'
 depends 'sudo'

diff --git a/...books/rubygems-app/recipes/deploy_user.rb → cookbooks/rubygems-people/recipes/deploy.rb b/...books/rubygems-app/recipes/deploy_user.rb → cookbooks/rubygems-people/recipes/deploy.rb
@@ -1,6 +1,6 @@
 #
-# Cookbook Name:: rubygems-app
-# Recipe:: deploy_user
+# Cookbook Name:: rubygems-people
+# Recipe:: deploy
 #
 
 include_recipe 'user'
@@ -13,6 +13,8 @@
   user['ssh_keys'].each { |k| keys << k } if user['deployer'] == true && user['environments'].include?(node.chef_environment)
 end
 
+keys << chef_vault_item('apps', 'shipit')['key']['public']
+
 user_account 'deploy' do
   comment   'Application Deployment'
   password  '$1$OghAZXA4$agFSQnk5/bJVs8rA8SaSh2'

diff --git a/cookbooks/rubygems-shipit/metadata.rb b/cookbooks/rubygems-shipit/metadata.rb
@@ -0,0 +1,12 @@
+name 'rubygems-shipit'
+
+version '0.0.44'
+
+depends 'apt'
+depends 'git'
+depends 'logrotate'
+depends 'nodejs'
+depends 'runit'
+depends 'sudo'
+
+supports 'ubuntu'
diff --git a/cookbooks/rubygems-shipit/recipes/app.rb b/cookbooks/rubygems-shipit/recipes/app.rb
@@ -0,0 +1,50 @@
+#
+# Cookbook Name:: rubygems-shipit
+# Recipe:: app
+#
+
+current_environment = 'production'
+current_app_path = '/applications/shipit/current'
+
+if File.directory? current_app_path
+
+  1.times do |n|
+    the_port = 3000 + n
+    service_name = "shipit-thin-#{the_port}"
+
+    runit_service service_name do
+      run_template_name 'thin'
+      default_logger true
+      options(
+        deploy_path: current_app_path,
+        run_as_user: 'shipit',
+        listen_ip: '127.0.0.1',
+        listen_port: the_port,
+        bundle_bin: '/usr/local/bin/bundle',
+        rack_env: current_environment,
+        timeout: 30
+      )
+      restart_on_update ::File.exist?("/etc/sv/#{service_name}")
+    end
+  end
+
+  1.times do |n|
+    worker_name = "shipit-resque-#{n+1}"
+
+    runit_service worker_name do
+      run_template_name 'resque'
+      default_logger true
+      restart_on_update false
+      options(
+        path: current_app_path,
+        user: 'shipit',
+        queue: 'deploys,default,*',
+        worker_name: worker_name,
+        num_workers: 1
+      )
+    end
+  end
+
+else
+  Chef::Log.error('You need to deploy the application before we can set up the application servers.')
+end
diff --git a/cookbooks/rubygems-shipit/recipes/default.rb b/cookbooks/rubygems-shipit/recipes/default.rb
@@ -0,0 +1,16 @@
+#
+# Cookbook Name:: rubygems-shipit
+# Recipe:: default
+#
+
+node.default['nodejs']['install_method'] = 'package'
+
+include_recipe 'apt'
+include_recipe 'git'
+include_recipe 'nodejs'
+include_recipe 'runit'
+
+include_recipe 'rubygems-people::deploy'
+include_recipe 'rubygems-shipit::setup'
+include_recipe 'rubygems-shipit::app'
+include_recipe 'rubygems-shipit::nginx'
diff --git a/cookbooks/rubygems-shipit/recipes/nginx.rb b/cookbooks/rubygems-shipit/recipes/nginx.rb
@@ -0,0 +1,43 @@
+#
+# Cookbook Name:: rubygems-shipit
+# Recipe:: nginx
+#
+
+node.default['nginx']['server_tokens'] = 'off'
+node.default['nginx']['default_site_enabled'] = false
+
+include_recipe 'nginx'
+
+directory "#{node['nginx']['dir']}/certs" do
+  owner 'root'
+  group 'root'
+  mode  '0644'
+end
+
+item = chef_vault_item('certs', 'production')
+
+file "#{node['nginx']['dir']}/certs/rubygems.org.key" do
+  content item['key']
+  owner  'root'
+  group  'root'
+  mode   '0644'
+  notifies :reload, 'service[nginx]'
+end
+
+file "#{node['nginx']['dir']}/certs/rubygems.org.crt" do
+  content item['crt']
+  owner  'root'
+  group  'root'
+  mode   '0644'
+  notifies :reload, 'service[nginx]'
+end
+
+template "#{node['nginx']['dir']}/sites-available/shipit" do
+  source 'nginx.conf.erb'
+  owner  'root'
+  group  'root'
+  mode   '0644'
+  notifies :reload, 'service[nginx]'
+end
+
+nginx_site 'shipit'