Merge branch 'fix_encoding' of https://github.com/brodock/gitlabhq into brodock-fix_encoding

Conflicts:
	lib/gitlab/encode.rb

app/controllers/refs_controller.rb

@@ -1,4 +1,5 @@
 class RefsController < ApplicationController
+  include Gitlabhq::Encode
   before_filter :project
 
   # Authorize
@@ -43,23 +44,26 @@ class RefsController < ApplicationController
         no_cache_headers
       end
     end
-  rescue
-    return render_404
   end
 
   def blob
     if @tree.is_blob?
+      if @tree.text?
+        encoding = detect_encoding(@tree.data)
+        mime_type = encoding ? "text/plain; charset=#{encoding}" : "text/plain"
+      else
+        mime_type = @tree.mime_type
+      end
+
       send_data(
         @tree.data,
-        :type => @tree.text? ? "text/plain" : @tree.mime_type,
+        :type => mime_type,
         :disposition => 'inline',
         :filename => @tree.name
       )
     else
       head(404)
     end
-  rescue
-    return render_404
   end
 
   def blame

app/views/commits/_commit.html.haml

@@ -8,7 +8,7 @@
       %strong.cgray= commit.author_name
       –
       = image_tag gravatar_icon(commit.author_email), :class => "avatar", :width => 16
-      %span.row_title= truncate(commit.safe_message, :length => 50) rescue "--broken encoding"
+      %span.row_title= truncate(commit.safe_message, :length => 50)
 
       %span.right.cgray
         = time_ago_in_words(commit.committed_date)

app/views/refs/_tree.html.haml

@@ -42,9 +42,9 @@
         .readme
           - if content.name =~ /\.(md|markdown)$/i
             = preserve do
-              = markdown(content.data.force_encoding('UTF-8'))
+              = markdown(content.data.detect_encoding!)
           - else
-            = simple_format(content.data.force_encoding('UTF-8'))
+            = simple_format(content.data.detect_encoding!)
 
 - if params[:path]
   - history_path = tree_file_project_ref_path(@project, @ref, params[:path])

app/views/refs/_tree_file.html.haml

@@ -13,7 +13,7 @@
       #tree-readme-holder
         .readme
           = preserve do
-            = markdown(file.data.force_encoding('UTF-8'))
+            = markdown(file.data.detect_encoding!)
     - else
       .view_file_content
         - unless file.empty?

lib/gitlab/encode.rb

@@ -1,3 +1,5 @@
+# Patch Strings to enable detect_encoding! on views
+require 'charlock_holmes/string'
 module Gitlab
   module Encode 
     extend self
@@ -5,16 +7,26 @@ module Gitlab
     def utf8 message
       return nil unless message
 
-      hash = CharlockHolmes::EncodingDetector.detect(message) rescue {}
+      detect = CharlockHolmes::EncodingDetector.detect(message) rescue {}
-      if hash[:encoding]
+
-        CharlockHolmes::Converter.convert(message, hash[:encoding], 'UTF-8')
+      # It's better to default to UTF-8 as sometimes it's wrongly detected as another charset
+      if detect[:encoding] && detect[:confidence] == 100
+        CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8')
       else
         message
       end.force_encoding("utf-8")
+
     # Prevent app from crash cause of 
     # encoding errors
     rescue
-      ""
+      "--broken encoding: #{encoding}"
+    end
+
+    def detect_encoding message
+      return nil unless message
+
+      hash = CharlockHolmes::EncodingDetector.detect(message) rescue {}
+      return hash[:encoding] ? hash[:encoding] : nil
     end
   end
 end