diff --git a/Gemfile b/Gemfile index fe37c5c8..8fef1106 100644 --- a/Gemfile +++ b/Gemfile @@ -3,28 +3,26 @@ source 'https://rubygems.org' ruby '2.1.6' gem 'travis-support', github: 'travis-ci/travis-support' -gem 'travis-config', '~> 1.0.0' +gem 'travis-config', '~> 1.0.0' gem 'sidekiq', '~> 4.0.0' gem 'redis-namespace' gem 'sentry-raven' -gem 'rollout', github: 'jamesgolick/rollout', :ref => 'v1.1.0' gem 'metriks' gem 'metriks-librato_metrics' gem 'jemalloc' -gem 'gh', github: 'travis-ci/gh' +gem 'gh' gem 'aws-sdk' gem 'actionmailer', '~> 3.2.18' gem 'roadie' gem 'roadie-rails', '~> 1.0' gem 'multi_json' -gem 'pusher', '~> 0.14.5' group :test do - gem 'rspec', '~> 2.14.0' - gem 'mocha', '~> 0.10.0' - gem 'webmock', '~> 1.8.0' + gem 'rspec', '~> 2.14.0' + gem 'mocha', '~> 0.10.0' + gem 'webmock', '~> 1.8.0' gem 'guard' gem 'guard-rspec' end diff --git a/Gemfile.lock b/Gemfile.lock index 8469e594..431eeb80 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,22 +1,3 @@ -GIT - remote: git://github.com/jamesgolick/rollout.git - revision: 3e7312cc018061b1ac6a8aeb9f11bbd0331da889 - ref: v1.1.0 - specs: - rollout (1.1.0) - -GIT - remote: git://github.com/travis-ci/gh.git - revision: 27e30fd01f6d5144d8d7d0c88db7875b95ed8939 - specs: - gh (0.14.0) - addressable - backports - faraday (~> 0.8) - multi_json (~> 1.0) - net-http-persistent (>= 2.7) - net-http-pipeline - GIT remote: git://github.com/travis-ci/travis-support.git revision: 2cd02d2a06fdd1e2fc2f129148c168b56f7c190f @@ -72,6 +53,13 @@ GEM foreman (0.78.0) thor (~> 0.19.1) formatador (0.2.5) + gh (0.14.0) + addressable + backports + faraday (~> 0.8) + multi_json (~> 1.0) + net-http-persistent (>= 2.7) + net-http-pipeline guard (2.13.0) formatador (>= 0.2.4) listen (>= 2.7, <= 4.0) @@ -87,7 +75,6 @@ GEM hashr (2.0.0) hike (1.2.3) hitimes (1.2.3) - httpclient (2.7.0.1) i18n (0.7.0) jemalloc (1.0.1) jmespath (1.1.3) @@ -127,11 +114,6 @@ GEM coderay (~> 1.1.0) method_source (~> 0.8.1) slop (~> 3.4) - pusher (0.14.6) - httpclient (~> 2.5) - multi_json (~> 1.0) - pusher-signature (~> 0.1.8) - pusher-signature (0.1.8) rack (1.4.7) rack-cache (1.5.1) rack (>= 0.4) @@ -202,7 +184,7 @@ DEPENDENCIES actionmailer (~> 3.2.18) aws-sdk foreman - gh! + gh guard guard-rspec jemalloc @@ -210,11 +192,9 @@ DEPENDENCIES metriks-librato_metrics mocha (~> 0.10.0) multi_json - pusher (~> 0.14.5) redis-namespace roadie roadie-rails (~> 1.0) - rollout! rspec (~> 2.14.0) sentry-raven sidekiq (~> 4.0.0) diff --git a/Procfile b/Procfile index b1f6b532..0c5e0d13 100644 --- a/Procfile +++ b/Procfile @@ -1 +1 @@ -tasks: bundle exec je sidekiq -c 25 -r ./lib/travis/tasks.rb -q campfire -q email -q flowdock -q github_commit_status -q github_status -q hipchat -q irc -q pusher -q sqwiggle -q webhook -q slack -q pushover +tasks: bundle exec je sidekiq -c 25 -r ./lib/travis/tasks.rb -q notifications -q campfire -q email -q flowdock -q github_commit_status -q github_status -q hipchat -q irc -q sqwiggle -q webhook -q slack -q pushover diff --git a/README.md b/README.md index 7721d514..3b255dc9 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Travis Task is a [Sidekiq](http://sidekiq.org/) based background processor whose These notifications are all queued up by state changes which are processed by [Travis Hub](https://github.com/travis-ci/travis-hub) and [Travis Gatekeeper](https://github.com/travis-ci/travis-gatekeeper). -And, to make Travis Tasks even more special, there is no database connection required! Travis Tasks is all about talking to 3rd party services, if it be [Pusher](http://pusher.com), [Mandrill](https://mandrillapp.com), [Campfire](http://campfirenow.com/), [Slack](http://slack.com/), or [Pushover](https://pushover.net/). +And, to make Travis Tasks even more special, there is no database connection required! Travis Tasks is all about talking to 3rd party services, if it be, [Mandrill](https://mandrillapp.com), [Campfire](http://campfirenow.com/), [Slack](http://slack.com/), or [Pushover](https://pushover.net/). You can find the full list of addon services Travis natively talks to within [Travis Core](https://github.com/travis-ci/travis-core/tree/master/lib/travis/addons). @@ -20,6 +20,3 @@ Please file any issues on the [central Travis CI issue tracker](https://github.c See LICENSE file. Copyright (c) 2011 [Travis CI development team](https://github.com/travis-ci). - - - diff --git a/lib/travis/addons.rb b/lib/travis/addons.rb index 2760e38b..2cfe5116 100644 --- a/lib/travis/addons.rb +++ b/lib/travis/addons.rb @@ -6,7 +6,6 @@ module Addons require 'travis/addons/github_status' require 'travis/addons/hipchat' require 'travis/addons/irc' - require 'travis/addons/pusher' require 'travis/addons/sqwiggle' require 'travis/addons/util' require 'travis/addons/webhook' diff --git a/lib/travis/addons/README.markdown b/lib/travis/addons/README.markdown index 936031d5..b58d3b18 100644 --- a/lib/travis/addons/README.markdown +++ b/lib/travis/addons/README.markdown @@ -8,10 +8,8 @@ The Addons are event handlers that accepts events such as "build finished" and f - GitHub Commit Statuses - Hipchat - IRC -- Pusher: Used to update our Web UI automatically. - Sqwiggle -- States cache: Caches the state of each branch in Memcached for status images. - Webhook - Pushover -To add a new notification service, an event handler and a task is needed. The event handler is run by [`travis-hub`](https://github.com/travis-ci/travis-hub) and has access to the database. This should check whether the event should be forwarded at all, and pull out any necessary configuration values. It should then asynchronously run the corresponding Task. The Task is run by [`travis-tasks`](https://github.com/travis-ci/travis-tasks) via Sidekiq and should do the actual API calls needed. The event handler should finish very quickly, while the task is allowed to take longer. +To add a new notification service, an event handler and a task is needed. The event handler is run by [`travis-hub`](https://github.com/travis-ci/travis-hub) and has access to the database. This should check whether the event should be forwarded at all, and pull out any necessary configuration values. It should then asynchronously run the corresponding Task. The Task is run by [`travis-tasks`](https://github.com/travis-ci/travis-tasks) via Sidekiq and should do the actual API calls needed. The event handler should finish very quickly, while the task is allowed to take longer. diff --git a/lib/travis/addons/email/task.rb b/lib/travis/addons/email/task.rb index 4dc0d075..6b0aa4ca 100644 --- a/lib/travis/addons/email/task.rb +++ b/lib/travis/addons/email/task.rb @@ -23,7 +23,7 @@ def type def process if recipients.any? Mailer::Build.finished_email(payload, recipients, broadcasts).deliver - info "status=sent msg='email sent' #{recipients.map { |r| 'email=' + obfuscate_email_address(r) }.join(' ')}" + info "type=email status=sent msg='email sent' #{recipients.map { |r| 'email=' + obfuscate_email_address(r) }.join(' ')}" end rescue Net::SMTPServerBusy => e error("Could not send email to: #{recipients} (error: #{e.message})") diff --git a/lib/travis/addons/github_status/task.rb b/lib/travis/addons/github_status/task.rb index ce641cc2..51cc7404 100644 --- a/lib/travis/addons/github_status/task.rb +++ b/lib/travis/addons/github_status/task.rb @@ -49,6 +49,12 @@ def process_with_token(token) GH.post(url, :state => state, :description => description, :target_url => target_url, :context => context) end rescue GH::Error(:response_status => 401) + error("type=github_status build=#{build[:id]} repo=#{repository[:slug]} state=#{state} commit=#{sha} response_status=401 reason=incorrect_auth") + nil + rescue GH::Error(:response_status => 403) + raise if Travis.config.env == 'production' + rescue GH::Error(:response_status => 404) + error("type=github_status build=#{build[:id]} repo=#{repository[:slug]} state=#{state} commit=#{sha} response_status=404 reason=repo_not_found_or_incorrect_auth") nil rescue GH::Error(:response_status => 422) error("type=github_status build=#{build[:id]} repo=#{repository[:slug]} state=#{state} commit=#{sha} response_status=422 reason=maximum_number_of_statuses") @@ -56,12 +62,7 @@ def process_with_token(token) rescue GH::Error => e message = "type=github_status build=#{build[:id]} repo=#{repository[:slug]} error=not_updated commit=#{sha} url=#{GH.api_host + url} message=#{e.message}" error(message) - response_status = e.info[:response_status] - case response_status - when 401, 422, 404 - else - raise message - end + raise message end def target_url @@ -95,7 +96,7 @@ def http_options(token) def headers { - "Accept" => "application/vnd.github.she-hulk-preview+json" + "Accept" => "application/vnd.github.v3+json" } end end diff --git a/lib/travis/addons/pusher.rb b/lib/travis/addons/pusher.rb deleted file mode 100644 index 68b586cf..00000000 --- a/lib/travis/addons/pusher.rb +++ /dev/null @@ -1,8 +0,0 @@ -module Travis - module Addons - module Pusher - require 'travis/addons/pusher/task' - end - end -end - diff --git a/lib/travis/addons/pusher/task.rb b/lib/travis/addons/pusher/task.rb deleted file mode 100644 index 5ba996cd..00000000 --- a/lib/travis/addons/pusher/task.rb +++ /dev/null @@ -1,100 +0,0 @@ -require 'travis/support/instrumentation' -require 'travis/support/chunkifier' - -module Travis - module Addons - module Pusher - - # Notifies registered clients about various state changes through Pusher. - class Task < Travis::Task - - def self.chunk_size - 9 * 1024 + 100 - end - - def event - params[:event] - end - - def client_event - @client_event ||= (event =~ /job:.*/ ? event.gsub(/(test|configure):/, '') : event) - end - - def channels - channels = private_channels? ? ["repo-#{repo_id}"] : ['common'] - channels.map { |channel| [channel_prefix, channels].compact.join('-') } - end - - private - - def process - channels.each { |channel| trigger(channel, payload) } - end - - def trigger(channel, payload) - parts(payload).each do |part| - begin - Travis.pusher[channel].trigger(client_event, part) - rescue ::Pusher::Error => e - Travis.logger.error("[addons:pusher] Could not send event due to Pusher::Error: #{e.message}, event=#{client_event}, payload: #{part.inspect}") - raise - end - end - end - - def job_id - payload[:id] - end - - def repo_id - # TODO api v1 is inconsistent here - payload.key?(:repository) ? payload[:repository][:id] : payload[:repository_id] - end - - def channel_prefix - 'private' if private_channels? - end - - def private_channels? - force_private_channels? || repository_private? - end - - def force_private_channels? - Travis.config.pusher.secure? - end - - def repository_private? - payload.key?(:repository) ? payload[:repository][:private] : payload[:repository_private] - end - - def parts(payload) - if client_event == 'job:log' && payload[:_log].present? - # split payload into 9kB chunks, the limit is 10 for entire request - # body, 1kB should be enough for headers - log = payload[:_log] - chunkifier = Chunkifier.new(log, chunk_size, :json => true) - - if chunkifier.length > 1 - # This should never happen when we update travis-worker to split log parts - # bigger than 9kB. - Travis.logger.warn("[addons:pusher] The log part from worker was bigger than 9kB (#{log.to_json.length}B), payload: #{payload.inspect}") - end - - chunkifier.each_with_index.map do |part, i| - new_payload = payload.dup.merge(:_log => part) - new_payload[:number] = "#{new_payload[:number]}.#{i}" unless i == 0 - new_payload[:final] = new_payload[:final] && chunkifier.length - 1 == i - new_payload - end - else - [payload] - end - end - - def chunk_size - self.class.chunk_size - end - end - end - end -end diff --git a/lib/travis/tasks/config.rb b/lib/travis/tasks/config.rb index 434d644e..a7c96024 100644 --- a/lib/travis/tasks/config.rb +++ b/lib/travis/tasks/config.rb @@ -1,22 +1,10 @@ require 'travis/config' -require 'pusher' module Travis def self.config @config ||= Tasks::Config.load end - def self.pusher - @pusher ||= ::Pusher.tap do |pusher| - pusher.scheme = config.pusher.scheme if config.pusher.scheme.present? - pusher.host = config.pusher.host if config.pusher.host.present? - pusher.port = config.pusher.port if config.pusher.port.present? - pusher.app_id = config.pusher.app_id - pusher.key = config.pusher.key - pusher.secret = config.pusher.secret - end - end - module Tasks class Config < Travis::Config HOSTS = { @@ -33,7 +21,6 @@ class Config < Travis::Config sidekiq: { namespace: "sidekiq", pool_size: 3 }, smtp: { }, ssl: { }, - pusher: { }, email: { }, assets: { host: HOSTS[Travis.env.to_sym] } diff --git a/lib/travis/tasks/middleware/logging.rb b/lib/travis/tasks/middleware/logging.rb index c850d9d8..8616823e 100644 --- a/lib/travis/tasks/middleware/logging.rb +++ b/lib/travis/tasks/middleware/logging.rb @@ -1,3 +1,5 @@ +require 'active_support/core_ext/string/inflections' + module Travis module Tasks module Middleware @@ -7,9 +9,10 @@ def call(worker, message, queue, &block) yield end ensure - uuid, _, _, payload, params = *message['args'] + uuid, notifier, _, payload, params = *message['args'] data = Hash.new.tap do |data| - data['type'] = queue + data['queue'] = queue + data['notifier'] = notifier.to_s.underscore.split('/')[2] if payload['build'] data['build'] = payload['build']['id'] elsif message['build_id'] diff --git a/spec/addons/github_status/task_spec.rb b/spec/addons/github_status/task_spec.rb index 81a805fb..ef17bff6 100644 --- a/spec/addons/github_status/task_spec.rb +++ b/spec/addons/github_status/task_spec.rb @@ -63,7 +63,7 @@ def run subject.new(payload, token: '12345').run end - it 'does not raise if a 422 error was raised by GH' do + it 'does not raise if a 422 error was returned by GH' do error = { response_status: 422 } GH.stubs(:post).raises(GH::Error.new('failed', nil, error)) expect { @@ -73,6 +73,16 @@ def run io.string.should include('reason=maximum_number_of_statuses') end + it 'does not raise if a 404 error was returned by GH' do + error = { response_status: 404 } + GH.stubs(:post).raises(GH::Error.new('failed', nil, error)) + expect { + run + }.not_to raise_error + io.string.should include('response_status=404') + io.string.should include('reason=repo_not_found_or_incorrect_auth') + end + describe 'logging' do it 'warns about a failed request' do GH.stubs(:post).raises(GH::Error.new(nil)) diff --git a/spec/addons/pusher/task_spec.rb b/spec/addons/pusher/task_spec.rb deleted file mode 100644 index 5f68a004..00000000 --- a/spec/addons/pusher/task_spec.rb +++ /dev/null @@ -1,163 +0,0 @@ -# encoding: utf-8 -require 'spec_helper' - -module Support - module Mocks - module Pusher - class Channel - attr_accessor :messages - - def initialize - @messages = [] - end - - def trigger(*args) - messages << args - end - alias :trigger_async :trigger - - def reset! - @messages = [] - end - alias :clear! :reset! - end - end - end -end - -describe Travis::Addons::Pusher::Task do - include Travis::Testing::Stubs - - let(:subject) { Travis::Addons::Pusher::Task } - let(:channel) { Support::Mocks::Pusher::Channel.new } - let(:task_payload) {Marshal.load(Marshal.dump(TASK_PAYLOAD))} - - before do - Travis.config.notifications = [:pusher] - Travis.pusher.stubs(:[]).returns(channel) - end - - def run(event, object, options = {}) - type = event.sub('test:', '').sub(':', '/') - payload = options[:params] ? task_payload.merge(options[:params]) : task_payload - subject.new(payload, event: event).run - end - - it 'logs Pusher errors and reraises' do - channel.expects(:trigger).raises(Pusher::Error.new('message')) - payload = task_payload.deep_symbolize_keys - Travis.logger.expects(:error).with("[addons:pusher] Could not send event due to Pusher::Error: message, event=job:started, payload: #{payload.inspect}") - - expect { - run('job:test:started', test) - }.to raise_error(Pusher::Error) - end - - describe 'run' do - it 'job:test:finished' do - run('job:test:finished', test) - channel.should have_message('job:finished', test) - end - - it 'build:finished' do - run('build:finished', build) - channel.should have_message('build:finished', build) - end - end - - describe 'channels' do - describe 'for a private repo' do - describe 'build event' do - let(:data) { { 'build' => { 'id' => 1, 'repository_id' => 1, }, 'repository' => { 'id' => 1 } } } - - before :each do - data['repository']['private'] = true - end - - it 'includes "private-repo-1" for the event "build:finished"' do - task = subject.new(data, :event => 'build:finished') - task.channels.should include("private-repo-#{repository.id}") - end - end - - describe 'job event' do - let(:data) { { 'id' => 1, 'build_id' => 1, 'repository_id' => 1 } } - - before :each do - data['repository_private'] = true - end - - it 'includes "private-repo-1" for the event "job:finished"' do - task = subject.new(data, :event => 'job:test:finished') - task.channels.should include("private-repo-#{repository.id}") - end - end - end - - describe 'for a public repo' do - describe 'with config.pusher.secure being false' do - before do - Travis.config.pusher.secure = false - end - - describe 'build event' do - let(:data) { { 'build' => { 'id' => 1, 'repository_id' => 1, }, 'repository' => { 'id' => 1 } } } - - before :each do - data['repository']['private'] = false - end - - it 'includes "repo-1" for the event "build:finished"' do - task = subject.new(data, :event => 'build:finished') - task.channels.should include('common') - end - end - - describe 'job event' do - let(:data) { { 'id' => 1, 'build_id' => 1, 'repository_id' => 1 } } - - before :each do - data['repository_private'] = false - end - - it 'includes "repo-1" for the event "job:finished"' do - task = subject.new(data, :event => 'job:test:finished') - task.channels.should include('common') - end - end - end - - describe 'with config.pusher.secure being true' do - before do - Travis.config.pusher.secure = true - end - - describe 'build event' do - let(:data) { { 'build' => { 'id' => 1, 'repository_id' => 1, }, 'repository' => { 'id' => 1 } } } - - before :each do - data['repository']['private'] = false - end - - it 'includes "private-repo-1" for the event "build:finished"' do - task = subject.new(data, :event => 'build:finished') - task.channels.should include("private-repo-#{repository.id}") - end - end - - describe 'build event' do - let(:data) { { 'id' => 1, 'build_id' => 1, 'repository_id' => 1 } } - - before :each do - data['repository_private'] = false - end - - it 'includes "private-repo-1" for the event "job:finished"' do - task = subject.new(data, :event => 'job:test:finished') - task.channels.should include("private-repo-#{repository.id}") - end - end - end - end - end -end