Merge pull request #2075 from getnikola/fix-2064

Kwpolska · Kwpolska · commit f0901409a4cc · 2015-09-12T20:01:12.000+02:00
add new normalize_html filter, make typogrify call it, fixes #2064
diff --git a/CHANGES.txt b/CHANGES.txt
@@ -3,6 +3,8 @@ New in master
 
 Features
 --------
+
+* New normalize_html filter
 * Support UTF-8 paths and encoded links when the ``USE_SLUGIFY`` option
   is disabled. (Issue #2037)
 * Per-document hyphenation using "hyphenate" metadata flag.
@@ -17,6 +19,7 @@ Features
 Bugfixes
 --------
 
+* Make typogrify filter work when applied from metadata (Issue #2064)
 * Handle metadata in post files that start with a BOM (Issue #2059)
 * Handle error downloading bootswatches (Issue #2054)
 * Monitor plugins/ in ``nikola auto`` (Issue #2044)
diff --git a/docs/manual.txt b/docs/manual.txt
@@ -1351,6 +1351,10 @@ typogrify_sans_widont
 minify_lines
    **THIS FILTER HAS BEEN TURNED INTO A NOOP** and currently does nothing.
 
+normalize_html
+   Pass HTML through LXML to normalize it. For example, it will resolve ``&quot;`` to actual
+   quotes. Usually not needed.
+
 yui_compressor
    Compress CSS/JavaScript using `YUI compressor <http://yui.github.io/yuicompressor/>`_
 
diff --git a/nikola/filters.py b/nikola/filters.py
@@ -34,6 +34,7 @@
 import tempfile
 import shlex
 
+import lxml
 try:
     import typogrify.filters as typo
 except ImportError:
@@ -236,6 +237,7 @@ def typogrify(data):
     if typo is None:
         req_missing(['typogrify'], 'use the typogrify filter')
 
+    data = _normalize_html(data)
     data = typo.amp(data)
     data = typo.widont(data)
     data = typo.smartypants(data)
@@ -253,6 +255,7 @@ def typogrify_sans_widont(data):
     if typo is None:
         req_missing(['typogrify'], 'use the typogrify_sans_widont filter')
 
+    data = _normalize_html(data)
     data = typo.amp(data)
     data = typo.smartypants(data)
     # Disabled because of typogrify bug where it breaks <title>
@@ -302,3 +305,15 @@ def jsminify(data):
     except Exception as exc:
         LOGGER.error("can't use javascript-minifier.com: {}", exc)
         return data
+
+
+def _normalize_html(data):
+    """Pass HTML through LXML to clean it up, if possible."""
+    try:
+        data = lxml.html.tostring(lxml.html.fragment_fromstring(data), encoding='unicode')
+    except:
+        pass
+    return data
+
+
+normalize_html = apply_to_text_file(_normalize_html)
diff --git a/nikola/plugins/task/posts.py b/nikola/plugins/task/posts.py
@@ -116,7 +116,7 @@ def tl_ch():
                             pass
                     else:
                         flist.append(f)
-                yield utils.apply_filters(task, {os.path.splitext(dest): flist})
+                yield utils.apply_filters(task, {os.path.splitext(dest)[-1]: flist})
 
     def dependence_on_timeline(self, post, lang):
         """Check if a post depends on the timeline."""
