html_to_unicode(): prioritize the BOM to determine the encoding (#191)

Author: BurnzZ

tests/test_encoding.py

@@ -220,13 +220,12 @@ def _assert_encoding_detected(
     def test_BOM(self):
         # utf-16 cases already tested, as is the BOM detection function
 
-        # http header takes precedence, irrespective of BOM
+        # BOM takes precedence, ahead of the http header
         bom_be_str = codecs.BOM_UTF16_BE + "hi".encode("utf-16-be")
-        expected = "\ufffd\ufffd\x00h\x00i"
-        self._assert_encoding("utf-8", bom_be_str, "utf-8", expected)
+        expected = "hi"
+        self._assert_encoding("utf-8", bom_be_str, "utf-16-be", expected)
 
-        # BOM is stripped when it agrees with the encoding, or used to
-        # determine encoding
+        # BOM is stripped when present
         bom_utf8_str = codecs.BOM_UTF8 + b"hi"
         self._assert_encoding("utf-8", bom_utf8_str, "utf-8", "hi")
         self._assert_encoding(None, bom_utf8_str, "utf-8", "hi")

w3lib/encoding.py

@@ -227,8 +227,8 @@ def html_to_unicode(
 
     It will try in order:
 
-    * http content type header
     * BOM (byte-order mark)
+    * http content type header
     * meta or xml tag declarations
     * auto-detection, if the `auto_detect_fun` keyword argument is not ``None``
     * default encoding in keyword arg (which defaults to utf8)
@@ -281,27 +281,16 @@ def html_to_unicode(
     >>>
 
     '''
-
-    enc = http_content_type_encoding(content_type_header)
     bom_enc, bom = read_bom(html_body_str)
-    if enc is not None:
-        # remove BOM if it agrees with the encoding
-        if enc == bom_enc:
-            bom = cast(bytes, bom)
-            html_body_str = html_body_str[len(bom) :]
-        elif enc == "utf-16" or enc == "utf-32":
-            # read endianness from BOM, or default to big endian
-            # tools.ietf.org/html/rfc2781 section 4.3
-            if bom_enc is not None and bom_enc.startswith(enc):
-                enc = bom_enc
-                bom = cast(bytes, bom)
-                html_body_str = html_body_str[len(bom) :]
-            else:
-                enc += "-be"
-        return enc, to_unicode(html_body_str, enc)
     if bom_enc is not None:
         bom = cast(bytes, bom)
         return bom_enc, to_unicode(html_body_str[len(bom) :], bom_enc)
+
+    enc = http_content_type_encoding(content_type_header)
+    if enc is not None:
+        if enc == "utf-16" or enc == "utf-32":
+            enc += "-be"
+        return enc, to_unicode(html_body_str, enc)
     enc = html_body_declared_encoding(html_body_str)
     if enc is None and (auto_detect_fun is not None):
         enc = auto_detect_fun(html_body_str)