Merge pull request #2578 from getnikola/wordpress-import-unescape-tags

felixfontein · web-flow · commit 3c45cd91ab00 · 2016-12-17T12:04:11.000+01:00
Unescaping HTML in WordPress tags and categories (fixes #2557).
diff --git a/CHANGES.txt b/CHANGES.txt
@@ -4,6 +4,8 @@ New in master
 Bugfixes
 --------
 
+* WordPress importer now correctly handles &amp; etc. in tags.
+  (Issue #2557)
 * If ``CODE_COLOR_SCHEME`` is empty, don’t generate ``code.css``
   (Issue #2597)
 * Don’t warn about ``nikolademo`` DISQUS account when comments are
diff --git a/nikola/plugins/command/import_wordpress.py b/nikola/plugins/command/import_wordpress.py
@@ -339,7 +339,7 @@ def _prepare(self, channel):
                 # cat_id = get_text_tag(cat, '{{{0}}}term_id'.format(wordpress_namespace), None)
                 cat_slug = get_text_tag(cat, '{{{0}}}category_nicename'.format(wordpress_namespace), None)
                 cat_parent_slug = get_text_tag(cat, '{{{0}}}category_parent'.format(wordpress_namespace), None)
-                cat_name = get_text_tag(cat, '{{{0}}}cat_name'.format(wordpress_namespace), None)
+                cat_name = utils.html_unescape(get_text_tag(cat, '{{{0}}}cat_name'.format(wordpress_namespace), None))
                 cat_path = [cat_name]
                 if cat_parent_slug in cat_map:
                     cat_path = cat_map[cat_parent_slug] + cat_path
@@ -824,16 +824,16 @@ def _create_metadata(self, status, excerpt, tags, categories, post_name=None):
                 if text in self._category_paths:
                     cats.append(self._category_paths[text])
                 else:
-                    cats.append(utils.join_hierarchical_category_path([text]))
+                    cats.append(utils.join_hierarchical_category_path([utils.html_unescape(text)]))
             other_meta['categories'] = ','.join(cats)
             if len(cats) > 0:
                 other_meta['category'] = cats[0]
                 if len(cats) > 1:
                     LOGGER.warn(('Post "{0}" has more than one category! ' +
                                  'Will only use the first one.').format(post_name))
-            tags_cats = tags
+            tags_cats = [utils.html_unescape(tag) for tag in tags]
         else:
-            tags_cats = tags + categories
+            tags_cats = [utils.html_unescape(tag) for tag in tags + categories]
         return tags_cats, other_meta
 
     _tag_sanitize_map = {True: {}, False: {}}
diff --git a/nikola/utils.py b/nikola/utils.py
@@ -94,7 +94,7 @@
            'NikolaPygmentsHTML', 'create_redirect', 'TreeNode',
            'flatten_tree_structure', 'parse_escaped_hierarchical_category_name',
            'join_hierarchical_category_path', 'clean_before_deployment', 'indent',
-           'load_data')
+           'load_data', 'html_unescape')
 
 # Are you looking for 'generic_rss_renderer'?
 # It's defined in nikola.nikola.Nikola (the site object).
@@ -1943,3 +1943,19 @@ def load_data(path):
         return
     with io.open(path, 'r', encoding='utf8') as inf:
         return loader.load(inf)
+
+
+# see http://stackoverflow.com/a/2087433
+try:
+    import html  # Python 3.4 and newer
+    html_unescape = html.unescape
+except (AttributeError, ImportError):
+    try:
+        from HTMLParser import HTMLParser  # Python 2.6 and 2.7
+    except ImportError:
+        from html.parser import HTMLParser  # Python 3 (up to 3.4)
+
+    def html_unescape(s):
+        """Convert all named and numeric character references  in the string s to the corresponding unicode characters."""
+        h = HTMLParser()
+        return h.unescape(s)
