Integrate Percy.io for visual regression tests (#3316)

* Integrate Percy.io for visual regression tests

* Configure Travis so Percy runs on latest ruby only.

* Wrap Percy::Capybara in a helper method

* Lock capybara and related testing gems to major versions

* Adjust Percy.io integration so PERCY_ENV=0 prevents Percy from being required

.travis.yml

@@ -6,8 +6,13 @@ services:
 before_install:
   - gem install bundler
   - gem update bundler
-rvm:
+matrix:
-  - 2.2.4
+  include:
-  - 2.3.0
+    - rvm: 2.2.4
-  - 2.4.0
+      env: "PERCY_ENABLE=0"
-  - jruby-9.1.6.0
+    - rvm: 2.3.0
+      env: "PERCY_ENABLE=0"
+    - rvm: 2.4.0
+      env: "PERCY_ENABLE=1"
+    - rvm: jruby-9.1.6.0
+      env: "PERCY_ENABLE=0"

sidekiq.gemspec

@@ -23,4 +23,10 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency      'minitest', '~> 5.10', '>= 5.10.1'
   gem.add_development_dependency      'rake', '~> 10.0'
   gem.add_development_dependency      'rails', '>= 3.2.0'
+
+  gem.add_development_dependency      'capybara', '~> 2.11'
+  gem.add_development_dependency      'poltergeist', '~> 1.12'
+  gem.add_development_dependency      'percy-capybara', '~> 2.3'
+  gem.add_development_dependency      'timecop', '~> 0.8'
+  gem.add_development_dependency      'mocha', '~> 1.1'
 end

test/helper.rb

@@ -3,6 +3,21 @@ $TESTING = true
 # disable minitest/parallel threads
 ENV["N"] = "0"
 
+require 'capybara'
+require 'capybara/dsl'
+require 'capybara/poltergeist'
+
+Capybara.register_driver :poltergeist do |app|
+  Capybara::Poltergeist::Driver.new(app,
+    debug: false, js_errors: false, timeout: 180
+  )
+end
+
+def percy_enabled?
+  !(ENV['PERCY_ENABLE'] == '0')
+end
+require 'percy/capybara' if percy_enabled?
+
 if ENV["COVERAGE"]
   require 'simplecov'
   SimpleCov.start do
@@ -73,3 +88,11 @@ def with_logging(lvl=Logger::DEBUG)
     Sidekiq.logger.level = old
   end
 end
+
+if percy_enabled?
+  # Initialize and finalize Percy.io
+  Percy::Capybara.initialize_build
+  MiniTest.after_run {
+    Percy::Capybara.finalize_build
+  }
+end

test/test_web.rb

@@ -3,10 +3,20 @@
 require_relative 'helper'
 require 'sidekiq/web'
 require 'rack/test'
+require 'timecop'
+require 'mocha/mini_test'
+
+# Tests in this file are a combination of:
+#   - Capybara tests (using visit) which render the web ui and check elements.
+#       The Capybara tests use Percy.io for visual regression testing.
+#   - Rack::Tests (using get & post) for increased speed and simplicity,
+#       when we're testing actions rather than UI presentation.
 
 class TestWeb < Sidekiq::Test
   describe 'sidekiq web' do
+
     include Rack::Test::Methods
+    include Capybara::DSL
 
     def app
       Sidekiq::Web
@@ -18,6 +28,20 @@ class TestWeb < Sidekiq::Test
 
     before do
       Sidekiq.redis {|c| c.flushdb }
+
+      Capybara.current_driver = :poltergeist
+      Capybara.app = app
+
+      # Freeze time so the time doesn't change in subsequent UI snapshots
+      Timecop.freeze(Time.utc(2016, 9, 1, 10, 5, 0))
+      # Stub redis_info so memory usage doesn't change in subsequent UI snapshots
+      Sidekiq.stubs(:redis_info).returns(Sidekiq::FAKE_INFO)
+    end
+
+    after do
+      Sidekiq.unstub(:redis_info)
+      Timecop.return
+      Capybara.use_default_driver
     end
 
     class WebWorker
@@ -34,24 +58,37 @@ class TestWeb < Sidekiq::Test
     end
 
     it 'can show text with any locales' do
-      rackenv = {'HTTP_ACCEPT_LANGUAGE' => 'ru,en'}
+      page.driver.headers = { 'Accept-Language' => 'ru,en' }
-      get '/', {}, rackenv
+      visit '/'
-      assert_match(/Панель управления/, last_response.body)
+      assert_text('Панель управления')
-      rackenv = {'HTTP_ACCEPT_LANGUAGE' => 'es,en'}
+      snapshot(page, name: 'Dashboard (Russian)')
-      get '/', {}, rackenv
+
-      assert_match(/Panel de Control/, last_response.body)
+      page.driver.headers = { 'Accept-Language' => 'es,en' }
-      rackenv = {'HTTP_ACCEPT_LANGUAGE' => 'en-us'}
+      visit '/'
-      get '/', {}, rackenv
+      assert_text('Panel de Control')
-      assert_match(/Dashboard/, last_response.body)
+      snapshot(page, name: 'Dashboard (Spanish)')
-      rackenv = {'HTTP_ACCEPT_LANGUAGE' => 'zh-cn'}
+
-      get '/', {}, rackenv
+      page.driver.headers = { 'Accept-Language' => 'en-us' }
-      assert_match(/信息板/, last_response.body)
+      visit '/'
-      rackenv = {'HTTP_ACCEPT_LANGUAGE' => 'zh-tw'}
+      assert_text('Dashboard')
-      get '/', {}, rackenv
+      snapshot(page, name: 'Dashboard (English)')
-      assert_match(/資訊主頁/, last_response.body)
+
-      rackenv = {'HTTP_ACCEPT_LANGUAGE' => 'nb'}
+      page.driver.headers = { 'Accept-Language' => 'zh-cn' }
-      get '/', {}, rackenv
+      visit '/'
-      assert_match(/Oversikt/, last_response.body)
+      assert_text('信息板')
+      snapshot(page, name: 'Dashboard (Chinese)')
+
+      page.driver.headers = { 'Accept-Language' => 'zh-tw' }
+      visit '/'
+      assert_text('資訊主頁')
+      snapshot(page, name: 'Dashboard (Taiwanese)')
+
+      page.driver.headers = { 'Accept-Language' => 'nb' }
+      visit '/'
+      assert_text('Oversikt')
+      snapshot(page, name: 'Dashboard (Norwegian)')
+
+      page.driver.headers = { 'Accept-Language' => 'en-us' }
     end
 
     describe 'busy' do
@@ -67,11 +104,21 @@ class TestWeb < Sidekiq::Test
         end
         assert_equal ['1001'], Sidekiq::Workers.new.map { |pid, tid, data| tid }
 
-        get '/busy'
+        visit '/busy'
-        assert_equal 200, last_response.status
+        assert_equal 200, page.status_code
-        assert_match(/status-active/, last_response.body)
+        assert_selector('.status-active')
-        assert_match(/critical/, last_response.body)
+        assert_text('critical')
-        assert_match(/WebWorker/, last_response.body)
+        assert_text('WebWorker')
+
+        assert has_button?('Quiet All')
+        assert has_button?('Stop All')
+
+        # Check the processes table has 2 rows - the header and process row
+        assert_equal 2, page.all('table.processes tr').count
+        assert has_button?('Quiet')
+        assert has_button?('Stop')
+
+        snapshot(page, name: 'Busy Page')
       end
 
       it 'can quiet a process' do
@@ -95,18 +142,21 @@ class TestWeb < Sidekiq::Test
       end
     end
 
+    describe 'queues' do
       it 'can display queues' do
         assert Sidekiq::Client.push('queue' => :foo, 'class' => WebWorker, 'args' => [1, 3])
 
-      get '/queues'
+        visit '/queues'
-      assert_equal 200, last_response.status
+        assert_equal 200, page.status_code
-      assert_match(/foo/, last_response.body)
+        assert_text('foo')
-      refute_match(/HardWorker/, last_response.body)
+        assert_no_text('HardWorker')
+        snapshot(page, name: 'Queues Page')
       end
 
       it 'handles queue view' do
-      get '/queues/default'
+        visit '/queues/default'
-      assert_equal 200, last_response.status
+        assert_equal 200, page.status_code
+        snapshot(page, name: 'Queue Page')
       end
 
       it 'can delete a queue' do
@@ -144,33 +194,38 @@ class TestWeb < Sidekiq::Test
           refute conn.lrange('queue:foo', 0, -1).include?("{\"foo\":\"bar\"}")
         end
       end
+    end
 
+
+    describe 'retries' do
       it 'can display retries' do
-      get '/retries'
+        visit '/retries'
-      assert_equal 200, last_response.status
+        assert_equal 200, page.status_code
-      assert_match(/found/, last_response.body)
+        assert_text('No retries were found')
-      refute_match(/HardWorker/, last_response.body)
+        assert_no_text('HardWorker')
+        snapshot(page, name: 'Retries Page - No Retries')
 
         add_retry
 
-      get '/retries'
+        visit '/retries'
-      assert_equal 200, last_response.status
+        assert_equal 200, page.status_code
-      refute_match(/found/, last_response.body)
+        assert_no_text('No retries were found')
-      assert_match(/HardWorker/, last_response.body)
+        assert_text('HardWorker')
+        snapshot(page, name: 'Retries Page - With Retry')
       end
 
       it 'can display a single retry' do
-      params = add_retry
+        params = add_retry 'abc'
-      get '/retries/0-shouldntexist'
+        visit "/retries/#{job_params(*params)}"
-      assert_equal 302, last_response.status
+        assert_equal 200, page.status_code
-      get "/retries/#{job_params(*params)}"
+        assert_text('HardWorker')
-      assert_equal 200, last_response.status
+        snapshot(page, name: 'Single Retry Page')
-      assert_match(/HardWorker/, last_response.body)
       end
 
-    it 'handles missing retry' do
+      it 'will redirect to retries when viewing a non-existant retry' do
-      get "/retries/0-shouldntexist"
+        get '/retries/0-shouldntexist'
         assert_equal 302, last_response.status
+        assert_equal 'http://example.org/retries', last_response.header['Location']
       end
 
       it 'can delete a single retry' do
@@ -215,32 +270,50 @@ class TestWeb < Sidekiq::Test
         assert_match(/#{params.first['args'][2]}/, last_response.body)
       end
 
-    it 'can display scheduled' do
+      it 'can retry all retries' do
-      get '/scheduled'
+        msg = add_retry.first
+        add_retry
+
+        post "/retries/all/retry", 'retry' => 'Retry'
+        assert_equal 302, last_response.status
+        assert_equal 'http://example.org/retries', last_response.header['Location']
+        assert_equal 2, Sidekiq::Queue.new("default").size
+
+        get '/queues/default'
         assert_equal 200, last_response.status
-      assert_match(/found/, last_response.body)
+        assert_match(/#{msg['args'][2]}/, last_response.body)
-      refute_match(/HardWorker/, last_response.body)
+      end
+    end
+
+    describe 'scheduled' do
+      it 'can display scheduled' do
+        visit '/scheduled'
+        assert_equal 200, page.status_code
+        assert_text('found')
+        assert_no_text('HardWorker')
+        snapshot(page, name: 'Scheduled Jobs Page - Nothing Scheduled')
 
         add_scheduled
 
-      get '/scheduled'
+        visit '/scheduled'
-      assert_equal 200, last_response.status
+        assert_equal 200, page.status_code
-      refute_match(/found/, last_response.body)
+        assert_no_text('found')
-      assert_match(/HardWorker/, last_response.body)
+        assert_text('HardWorker')
+        snapshot(page, name: 'Scheduled Jobs Page - Job Scheduled')
       end
 
       it 'can display a single scheduled job' do
-      params = add_scheduled
+        params = add_scheduled 'abc'
-      get '/scheduled/0-shouldntexist'
+        visit "/scheduled/#{job_params(*params)}"
-      assert_equal 302, last_response.status
+        assert_equal 200, page.status_code
-      get "/scheduled/#{job_params(*params)}"
+        assert_text 'HardWorker'
-      assert_equal 200, last_response.status
+        snapshot(page, name: 'Scheduled Job Page')
-      assert_match(/HardWorker/, last_response.body)
       end
 
       it 'handles missing scheduled job' do
         get "/scheduled/0-shouldntexist"
         assert_equal 302, last_response.status
+        assert_equal 'http://example.org/scheduled', last_response.header['Location']
       end
 
       it 'can add to queue a single scheduled job' do
@@ -292,19 +365,6 @@ class TestWeb < Sidekiq::Test
           assert_match(/#{params[0]['args'][2]}/, last_response.body)
         end
       end
-
-    it 'can retry all retries' do
-      msg = add_retry.first
-      add_retry
-
-      post "/retries/all/retry", 'retry' => 'Retry'
-      assert_equal 302, last_response.status
-      assert_equal 'http://example.org/retries', last_response.header['Location']
-      assert_equal 2, Sidekiq::Queue.new("default").size
-
-      get '/queues/default'
-      assert_equal 200, last_response.status
-      assert_match(/#{msg['args'][2]}/, last_response.body)
     end
 
     it 'calls updatePage() once when polling' do
@@ -463,15 +523,17 @@ class TestWeb < Sidekiq::Test
 
     describe 'dead jobs' do
       it 'shows empty index' do
-        get 'morgue'
+        visit '/morgue'
-        assert_equal 200, last_response.status
+        assert_equal 200, page.status_code
+        snapshot(page, name: 'Dead Jobs Page - Empty')
       end
 
       it 'shows index with jobs' do
         (_, score) = add_dead
-        get 'morgue'
+        visit '/morgue'
-        assert_equal 200, last_response.status
+        assert_equal 200, page.status_code
-        assert_match(/#{score}/, last_response.body)
+        assert_text(score.to_i)
+        snapshot(page, name: 'Dead Jobs Page - With Job')
       end
 
       it 'can delete all dead' do
@@ -496,18 +558,18 @@ class TestWeb < Sidekiq::Test
       end
     end
 
-    def add_scheduled
+    def add_scheduled(job_id=SecureRandom.hex(12))
       score = Time.now.to_f
       msg = { 'class' => 'HardWorker',
               'args' => ['bob', 1, Time.now.to_f],
-              'jid' => SecureRandom.hex(12) }
+              'jid' => job_id }
       Sidekiq.redis do |conn|
         conn.zadd('schedule', score, Sidekiq.dump_json(msg))
       end
       [msg, score]
     end
 
-    def add_retry
+    def add_retry(job_id=SecureRandom.hex(12))
       msg = { 'class' => 'HardWorker',
               'args' => ['bob', 1, Time.now.to_f],
               'queue' => 'default',
@@ -515,7 +577,7 @@ class TestWeb < Sidekiq::Test
               'error_class' => 'RuntimeError',
               'retry_count' => 0,
               'failed_at' => Time.now.to_f,
-              'jid' => SecureRandom.hex(12) }
+              'jid' => job_id }
       score = Time.now.to_f
       Sidekiq.redis do |conn|
         conn.zadd('retry', score, Sidekiq.dump_json(msg))
@@ -524,7 +586,7 @@ class TestWeb < Sidekiq::Test
       [msg, score]
     end
 
-    def add_dead
+    def add_dead(job_id=SecureRandom.hex(12))
       msg = { 'class' => 'HardWorker',
               'args' => ['bob', 1, Time.now.to_f],
               'queue' => 'foo',
@@ -532,7 +594,7 @@ class TestWeb < Sidekiq::Test
               'error_class' => 'RuntimeError',
               'retry_count' => 0,
               'failed_at' => Time.now.utc,
-              'jid' => SecureRandom.hex(12) }
+              'jid' => job_id }
       score = Time.now.to_f
       Sidekiq.redis do |conn|
         conn.zadd('dead', score, Sidekiq.dump_json(msg))
@@ -540,7 +602,7 @@ class TestWeb < Sidekiq::Test
       [msg, score]
     end
 
-    def add_xss_retry
+    def add_xss_retry(job_id=SecureRandom.hex(12))
       msg = { 'class' => 'FailWorker',
               'args' => ['<a>hello</a>'],
               'queue' => 'foo',
@@ -548,7 +610,7 @@ class TestWeb < Sidekiq::Test
               'error_class' => 'RuntimeError',
               'retry_count' => 0,
               'failed_at' => Time.now.to_f,
-              'jid' => SecureRandom.hex(12) }
+              'jid' => job_id }
       score = Time.now.to_f
       Sidekiq.redis do |conn|
         conn.zadd('retry', score, Sidekiq.dump_json(msg))
@@ -568,112 +630,9 @@ class TestWeb < Sidekiq::Test
         end
       end
     end
-  end
 
-  describe 'sidekiq web with basic auth' do
+    def snapshot(page, options)
-    include Rack::Test::Methods
+      Percy::Capybara.snapshot(page, options) if percy_enabled?
-
-    def app
-      app = Sidekiq::Web.new
-      app.use(Rack::Auth::Basic) { |user, pass| user == "a" && pass == "b" }
-
-      app
-    end
-
-    it 'requires basic authentication' do
-      get '/'
-
-      assert_equal 401, last_response.status
-      refute_nil last_response.header["WWW-Authenticate"]
-    end
-
-    it 'authenticates successfuly' do
-      basic_authorize 'a', 'b'
-
-      get '/'
-
-      assert_equal 200, last_response.status
-    end
-  end
-
-  describe 'sidekiq web with custom session' do
-    include Rack::Test::Methods
-
-    def app
-      app = Sidekiq::Web.new
-
-      app.use Rack::Session::Cookie, secret: 'v3rys3cr31', host: 'nicehost.org'
-
-      app
-    end
-
-    it 'requires basic authentication' do
-      get '/'
-
-      session_options = last_request.env['rack.session'].options
-
-      assert_equal 'v3rys3cr31', session_options[:secret]
-      assert_equal 'nicehost.org', session_options[:host]
-    end
-  end
-
-  describe 'sidekiq web sessions options' do
-    include Rack::Test::Methods
-
-    describe 'using #disable' do
-      def app
-        app = Sidekiq::Web.new
-        app.disable(:sessions)
-        app
-      end
-
-      it "doesn't create sessions" do
-        get '/'
-        assert_nil last_request.env['rack.session']
-      end
-    end
-
-    describe 'using #set with false argument' do
-      def app
-        app = Sidekiq::Web.new
-        app.set(:sessions, false)
-        app
-      end
-
-      it "doesn't create sessions" do
-        get '/'
-        assert_nil last_request.env['rack.session']
-      end
-    end
-
-    describe 'using #set with an hash' do
-      def app
-        app = Sidekiq::Web.new
-        app.set(:sessions, { domain: :all })
-        app
-      end
-
-      it "creates sessions" do
-        get '/'
-        refute_nil   last_request.env['rack.session']
-        refute_empty last_request.env['rack.session'].options
-        assert_equal :all, last_request.env['rack.session'].options[:domain]
-      end
-    end
-
-    describe 'using #enable' do
-      def app
-        app = Sidekiq::Web.new
-        app.enable(:sessions)
-        app
-      end
-
-      it "creates sessions" do
-        get '/'
-        refute_nil   last_request.env['rack.session']
-        refute_empty last_request.env['rack.session'].options
-        refute_nil   last_request.env['rack.session'].options[:secret]
-      end
     end
   end
 end

test/test_web_auth.rb

@@ -0,0 +1,54 @@
+# encoding: utf-8
+# frozen_string_literal: true
+require_relative 'helper'
+require 'sidekiq/web'
+require 'rack/test'
+
+class TestWebAuth < Sidekiq::Test
+  describe 'sidekiq web with basic auth' do
+    include Rack::Test::Methods
+
+    def app
+      app = Sidekiq::Web.new
+      app.use(Rack::Auth::Basic) { |user, pass| user == "a" && pass == "b" }
+
+      app
+    end
+
+    it 'requires basic authentication' do
+      get '/'
+
+      assert_equal 401, last_response.status
+      refute_nil last_response.header["WWW-Authenticate"]
+    end
+
+    it 'authenticates successfuly' do
+      basic_authorize 'a', 'b'
+
+      get '/'
+
+      assert_equal 200, last_response.status
+    end
+  end
+
+  describe 'sidekiq web with custom session' do
+    include Rack::Test::Methods
+
+    def app
+      app = Sidekiq::Web.new
+
+      app.use Rack::Session::Cookie, secret: 'v3rys3cr31', host: 'nicehost.org'
+
+      app
+    end
+
+    it 'requires basic authentication' do
+      get '/'
+
+      session_options = last_request.env['rack.session'].options
+
+      assert_equal 'v3rys3cr31', session_options[:secret]
+      assert_equal 'nicehost.org', session_options[:host]
+    end
+  end
+end

test/test_web_sessions.rb

@@ -0,0 +1,67 @@
+# encoding: utf-8
+# frozen_string_literal: true
+require_relative 'helper'
+require 'sidekiq/web'
+require 'rack/test'
+
+class TestWebSessions < Sidekiq::Test
+  describe 'sidekiq web sessions options' do
+    include Rack::Test::Methods
+
+    describe 'using #disable' do
+      def app
+        app = Sidekiq::Web.new
+        app.disable(:sessions)
+        app
+      end
+
+      it "doesn't create sessions" do
+        get '/'
+        assert_nil last_request.env['rack.session']
+      end
+    end
+
+    describe 'using #set with false argument' do
+      def app
+        app = Sidekiq::Web.new
+        app.set(:sessions, false)
+        app
+      end
+
+      it "doesn't create sessions" do
+        get '/'
+        assert_nil last_request.env['rack.session']
+      end
+    end
+
+    describe 'using #set with an hash' do
+      def app
+        app = Sidekiq::Web.new
+        app.set(:sessions, { domain: :all })
+        app
+      end
+
+      it "creates sessions" do
+        get '/'
+        refute_nil   last_request.env['rack.session']
+        refute_empty last_request.env['rack.session'].options
+        assert_equal :all, last_request.env['rack.session'].options[:domain]
+      end
+    end
+
+    describe 'using #enable' do
+      def app
+        app = Sidekiq::Web.new
+        app.enable(:sessions)
+        app
+      end
+
+      it "creates sessions" do
+        get '/'
+        refute_nil   last_request.env['rack.session']
+        refute_empty last_request.env['rack.session'].options
+        refute_nil   last_request.env['rack.session'].options[:secret]
+      end
+    end
+  end
+end