Have everything support Unicode BOMs.

doc-src/HAML_CHANGELOG.md

@@ -80,9 +80,12 @@ of the many and varied [Haml implementations](http://en.wikipedia.org/wiki/Haml#
 
 ### Ruby 1.9 Support
 
-Haml and `html2haml` now produce more descriptive errors
+* Haml and `html2haml` now produce more descriptive errors
-when given a template with invalid byte sequences for that template's encoding,
+  when given a template with invalid byte sequences for that template's encoding,
-including the line number and the offending character.
+  including the line number and the offending character.
+
+* Haml and `html2haml` now accept Unicode documents with a
+  [byte-order-mark](http://en.wikipedia.org/wiki/Byte_order_mark).
 
 ### `:css` Filter
 

doc-src/SASS_CHANGELOG.md

@@ -162,9 +162,12 @@ That is, with the nested properties indented in the source.
 
 ### Ruby 1.9 Support
 
-Sass and `css2sass` now produce more descriptive errors
+* Sass and `css2sass` now produce more descriptive errors
-when given a template with invalid byte sequences for that template's encoding,
+  when given a template with invalid byte sequences for that template's encoding,
-including the line number and the offending character.
+  including the line number and the offending character.
+
+* Sass and `css2sass` now accept Unicode documents with a
+  [byte-order-mark](http://en.wikipedia.org/wiki/Byte_order_mark).
 
 ### Rack Support
 

lib/haml/engine.rb

@@ -99,7 +99,7 @@ module Haml
         @options[:encoding] = @options[:encoding].name
       end
 
-      check_encoding(template) {|msg, line| raise Haml::Error.new(msg, line)}
+      template = check_encoding(template) {|msg, line| raise Haml::Error.new(msg, line)}
 
       # :eod is a special end-of-document marker
       @template = (template.rstrip).split(/\r\n|\r|\n/) + [:eod, :eod]

lib/haml/html.rb

@@ -127,7 +127,7 @@ module Haml
           template = template.read
         end
 
-        Haml::Util.check_encoding(template) {|msg, line| raise Haml::Error.new(msg, line)}
+        template = Haml::Util.check_encoding(template) {|msg, line| raise Haml::Error.new(msg, line)}
 
         if @options[:erb]
           require 'haml/html/erb'

lib/haml/util.rb

@@ -237,7 +237,8 @@ module Haml
       Haml::Util::RUBY_VERSION[0] == 1 && Haml::Util::RUBY_VERSION[1] < 9
     end
 
-    # Checks that the encoding of a string is valid in Ruby 1.9.
+    # Checks that the encoding of a string is valid in Ruby 1.9
+    # and cleans up potential encoding gotchas like the UTF-8 BOM.
     # If it's not, yields an error string describing the invalid character
     # and the line on which it occurrs.
     #
@@ -245,9 +246,19 @@ module Haml
     # @yield [msg] A block in which an encoding error can be raised.
     #   Only yields if there is an encoding error
     # @yieldparam msg [String] The error message to be raised
+    # @return [String] `str`, potentially with encoding gotchas like BOMs removed
     def check_encoding(str)
-      return if ruby1_8?
+      if ruby1_8?
-      return if str.valid_encoding?
+        return str.gsub(/\A\xEF\xBB\xBF/, '') # Get rid of the UTF-8 BOM
+      elsif str.valid_encoding?
+        # Get rid of the Unicode BOM if possible
+        if str.encoding.name =~ /^UTF-(8|16|32)(BE|LE)?$/
+          return str.gsub(Regexp.new("\\A\uFEFF".encode(str.encoding.name)), '')
+        else
+          return str
+        end
+      end
+
       encoding = str.encoding
       newlines = Regexp.new("\r\n|\r|\n".encode(encoding).force_encoding("binary"))
       str.force_encoding("binary").split(newlines).each_with_index do |line, i|
@@ -259,6 +270,7 @@ Invalid #{encoding.name} character #{e.error_char.dump}
 MSG
         end
       end
+      return str
     end
 
     # Checks to see if a class has a given method.

lib/sass/css.rb

@@ -76,7 +76,7 @@ module Sass
       @options = options.dup
       # Backwards compatibility
       @options[:old] = true if @options[:alternate] == false
-      @template = StringScanner.new(template)
+      @template_str = template
     end
 
     # Converts the CSS template into Sass code.
@@ -84,10 +84,10 @@ module Sass
     # @return [String] The resulting Sass code
     # @raise [Sass::SyntaxError] if there's an error parsing the CSS template
     def render
-      Haml::Util.check_encoding(@template.string) do |msg, line|
+      @template = StringScanner.new(
-        raise Sass::SyntaxError.new(msg, :line => line)
+        Haml::Util.check_encoding(@template_str) do |msg, line|
-      end
+          raise Sass::SyntaxError.new(msg, :line => line)
-
+        end)
       build_tree.to_sass(0, @options).strip + "\n"
     rescue Sass::SyntaxError => err
       err.modify_backtrace(:filename => @options[:filename] || '(css)', :line => @line)

lib/sass/engine.rb

@@ -172,7 +172,7 @@ module Sass
     # @return [Sass::Tree::Node] The root of the parse tree.
     # @raise [Sass::SyntaxError] if there's an error in the document
     def to_tree
-      check_encoding(@template) {|msg, line| raise Sass::SyntaxError.new(msg, :line => line)}
+      @template = check_encoding(@template) {|msg, line| raise Sass::SyntaxError.new(msg, :line => line)}
 
       root = Tree::RootNode.new(@template)
       append_children(root, tree(tabulate(@template)).first, true)

test/haml/engine_test.rb

@@ -1250,6 +1250,17 @@ SASS
 
   # Encodings
 
+  def test_utf_8_bom
+    assert_equal <<CSS, render(<<SCSS)
+<div class='foo'>
+  <p>baz</p>
+</div>
+CSS
+\xEF\xBB\xBF.foo
+  %p baz
+SCSS
+  end
+
   unless Haml::Util.ruby1_8?
     def test_default_encoding
       assert_equal(Encoding.find("utf-8"), render(<<HAML.encode("us-ascii")).encoding)