Improve parsing of HTTP_ACCEPT_LANGUAGE (continue Nate's work) (#3449)

* Add test case for sv locale

* Use Rack::Utils to parse locale header

* Take "q" value into account

* Make '*' match the default locale.

* Add test for available_locales

* Correct test case sv -> en

* Add missing test cases for Safari requests

* Add missing require needed to run a single test file

* Reimplement WebHelpers#locale to handle regions in header

Implementation inspired by:

https://github.com/iain/http_accept_language/blob/master/lib/http_accept_language/parser.rb

Also see:

https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.4

* Add docs and references

* Add failing test cases for pt-br, pt-pt, pt (examples taken from Chrome & Safari)

* Add more test cases for Mac + Chrome + UK English + US English

* Make test cases for 'pt-PT,pt;q=0.8,en-US;q=0.6,en;q=0.4' and 'pt-pt' pass

* Make special case 'ru,en' work (equal qvalues)

lib/sidekiq/web/helpers.rb

@@ -24,6 +24,7 @@ module Sidekiq
     def clear_caches
       @@strings = nil
       @@locale_files = nil
+      @@available_locales = nil
     end
 
     def locale_files
@@ -32,6 +33,10 @@ module Sidekiq
       end
     end
 
+    def available_locales
+      @@available_locales ||= locale_files.map { |path| File.basename(path, '.yml') }.uniq
+    end
+
     def find_locale_files(lang)
       locale_files.select { |file| file =~ /\/#{lang}\.yml$/ }
     end
@@ -73,20 +78,36 @@ module Sidekiq
       text_direction == 'rtl'
     end
 
-    # Given a browser request Accept-Language header like
-    # "fr-FR,fr;q=0.8,en-US;q=0.6,en;q=0.4,ru;q=0.2", this function
-    # will return "fr" since that's the first code with a matching
-    # locale in web/locales
+    # See https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.4
+    def user_preferred_languages
+      languages = env['HTTP_ACCEPT_LANGUAGE'.freeze]
+      languages.to_s.downcase.gsub(/\s+/, '').split(',').map do |language|
+        locale, quality = language.split(';q=', 2)
+        locale  = nil if locale == '*' # Ignore wildcards
+        quality = quality ? quality.to_f : 1.0
+        [locale, quality]
+      end.sort do |(_, left), (_, right)|
+        right <=> left
+      end.map(&:first).compact
+    end
+
+    # Given an Accept-Language header like "fr-FR,fr;q=0.8,en-US;q=0.6,en;q=0.4,ru;q=0.2"
+    # this method will try to best match the available locales to the user's preferred languages.
+    #
+    # Inspiration taken from https://github.com/iain/http_accept_language/blob/master/lib/http_accept_language/parser.rb
     def locale
       @locale ||= begin
-        locale = 'en'.freeze
-        languages = env['HTTP_ACCEPT_LANGUAGE'.freeze] || 'en'.freeze
-        languages.downcase.split(','.freeze).each do |lang|
-          next if lang == '*'.freeze
-          lang = lang.split(';'.freeze)[0]
-          break locale = lang if find_locale_files(lang).any?
-        end
-        locale
+        matched_locale = user_preferred_languages.map do |preferred|
+          preferred_language = preferred.split('-', 2).first
+
+          lang_group = available_locales.select do |available|
+            preferred_language == available.split('-', 2).first
+          end
+
+          lang_group.find { |lang| lang == preferred } || lang_group.min_by(&:length)
+        end.compact.first
+
+        matched_locale || 'en'
       end
     end
 

test/test_web_helpers.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 require_relative 'helper'
+require 'sidekiq/web'
 
 class TestWebHelpers < Sidekiq::Test
 
@@ -42,13 +43,56 @@ class TestWebHelpers < Sidekiq::Test
     obj = Helpers.new('HTTP_ACCEPT_LANGUAGE' => 'zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4,ru;q=0.2')
     assert_equal 'zh-cn', obj.locale
 
+    obj = Helpers.new('HTTP_ACCEPT_LANGUAGE' => 'en-US,sv-SE;q=0.8,sv;q=0.6,en;q=0.4')
+    assert_equal 'en', obj.locale
+
     obj = Helpers.new('HTTP_ACCEPT_LANGUAGE' => 'nb-NO,nb;q=0.2')
     assert_equal 'nb', obj.locale
 
+    obj = Helpers.new('HTTP_ACCEPT_LANGUAGE' => 'en-us')
+    assert_equal 'en', obj.locale
+
+    obj = Helpers.new('HTTP_ACCEPT_LANGUAGE' => 'sv-se')
+    assert_equal 'sv', obj.locale
+
+    obj = Helpers.new('HTTP_ACCEPT_LANGUAGE' => 'pt-BR,pt;q=0.8,en-US;q=0.6,en;q=0.4')
+    assert_equal 'pt-br', obj.locale
+
+    obj = Helpers.new('HTTP_ACCEPT_LANGUAGE' => 'pt-PT,pt;q=0.8,en-US;q=0.6,en;q=0.4')
+    assert_equal 'pt', obj.locale
+
+    obj = Helpers.new('HTTP_ACCEPT_LANGUAGE' => 'pt-br')
+    assert_equal 'pt-br', obj.locale
+
+    obj = Helpers.new('HTTP_ACCEPT_LANGUAGE' => 'pt-pt')
+    assert_equal 'pt', obj.locale
+
+    obj = Helpers.new('HTTP_ACCEPT_LANGUAGE' => 'pt')
+    assert_equal 'pt', obj.locale
+
     obj = Helpers.new('HTTP_ACCEPT_LANGUAGE' => 'en-us; *')
     assert_equal 'en', obj.locale
 
+    obj = Helpers.new('HTTP_ACCEPT_LANGUAGE' => 'en-US,en;q=0.8')
+    assert_equal 'en', obj.locale
+
+    obj = Helpers.new('HTTP_ACCEPT_LANGUAGE' => 'en-GB,en-US;q=0.8,en;q=0.6')
+    assert_equal 'en', obj.locale
+
+    obj = Helpers.new('HTTP_ACCEPT_LANGUAGE' => 'ru,en')
+    assert_equal 'ru', obj.locale
+
     obj = Helpers.new('HTTP_ACCEPT_LANGUAGE' => '*')
     assert_equal 'en', obj.locale
   end
+
+  def test_available_locales
+    obj = Helpers.new
+    expected = %w(
+      ar cs da de el en es fa fr he hi it ja
+      ko nb nl pl pt-br pt ru sv ta uk ur
+      zh-cn zh-tw
+    )
+    assert_equal expected, obj.available_locales
+  end
 end