Merge pull request #2623 from getnikola/rest-compile_string-shortcodes

Move shortcode handling to reST compiler’s compile_string
getnikola · Jan 8, 2017 · e43bf26 · e43bf26
2 parents f21cd80 + 21d63e8
commit e43bf26
Show file tree

Hide file tree

Showing 10 changed files with 57 additions and 19 deletions.
diff --git a/CHANGES.txt b/CHANGES.txt
@@ -1,6 +1,11 @@
 New in master
 =============
 
+**Minor API change:** The ``compile_string`` compiler method (partially
+internal) now takes a post argument and returns between two and four
+values, adding ``shortcode_deps`` and shortcode support. See issues
+#2623 and #2624.
+
 Bugfixes
 --------
 

diff --git a/nikola/plugin_categories.py b/nikola/plugin_categories.py
@@ -303,6 +303,14 @@ def compile(self, source, dest, is_two_file=True, post=None, lang=None):
         # ignore `compile_html`
         self.compile_html(source, dest, is_two_file)
 
+    def compile_string(self, data, source_path=None, is_two_file=True, post=None, lang=None):
+        """Compile the source file into HTML strings (with shortcode support).
+
+        Returns a tuple of at least two elements: HTML string [0] and shortcode dependencies [last].
+        """
+        # This function used to have some different APIs in different places.
+        raise NotImplementedError()
+
     # TODO remove in v8
     def compile_html(self, source, dest, is_two_file=True):
         """Compile the source, save it on dest (DEPRECATED)."""

diff --git a/nikola/plugins/command/rst2html/__init__.py b/nikola/plugins/command/rst2html/__init__.py
@@ -52,7 +52,7 @@ def _execute(self, options, args):
         source = args[0]
         with io.open(source, "r", encoding="utf8") as in_file:
             data = in_file.read()
-            output, error_level, deps = compiler.compile_string(data, source, True)
+            output, error_level, deps, shortcode_deps = compiler.compile_string(data, source, True)
 
         rstcss_path = resource_filename('nikola', 'data/themes/base/assets/css/rst.css')
         with io.open(rstcss_path, "r", encoding="utf8") as fh:

diff --git a/nikola/plugins/compile/html.py b/nikola/plugins/compile/html.py
@@ -41,15 +41,19 @@ class CompileHtml(PageCompiler):
     name = "html"
     friendly_name = "HTML"
 
+    def compile_string(self, data, source_path=None, is_two_file=True, post=None, lang=None):
+        """Compile HTML into HTML strings, with shortcode support."""
+        if not is_two_file:
+            _, data = self.split_metadata(data)
+        return self.site.apply_shortcodes(data, with_dependencies=True, extra_context={'post': post})
+
     def compile(self, source, dest, is_two_file=True, post=None, lang=None):
         """Compile the source file into HTML and save as dest."""
         makedirs(os.path.dirname(dest))
         with io.open(dest, "w+", encoding="utf8") as out_file:
             with io.open(source, "r", encoding="utf8") as in_file:
                 data = in_file.read()
-            if not is_two_file:
-                _, data = self.split_metadata(data)
-            data, shortcode_deps = self.site.apply_shortcodes(data, with_dependencies=True, extra_context=dict(post=post))
+            data, shortcode_deps = self.compile_string(data, source, is_two_file, post, lang)
             out_file.write(data)
         if post is None:
             if shortcode_deps:

diff --git a/nikola/plugins/compile/ipynb.py b/nikola/plugins/compile/ipynb.py
@@ -77,28 +77,38 @@ def set_site(self, site):
         self.logger = get_logger('compile_ipynb', STDERR_HANDLER)
         super(CompileIPynb, self).set_site(site)
 
-    def compile_string(self, source, is_two_file=True):
+    def _compile_string(self, nb_json):
         """Export notebooks as HTML strings."""
         if flag is None:
             req_missing(['ipython[notebook]>=2.0.0'], 'build this site (compile ipynb)')
         c = Config(self.site.config['IPYNB_CONFIG'])
         exportHtml = HTMLExporter(config=c)
-        with io.open(source, "r", encoding="utf8") as in_file:
-            nb_json = nbformat.read(in_file, current_nbformat)
-        (body, resources) = exportHtml.from_notebook_node(nb_json)
+        body, _ = exportHtml.from_notebook_node(nb_json)
         return body
 
+    @staticmethod
+    def _nbformat_read(in_file):
+        return nbformat.read(in_file, current_nbformat)
+
+    def compile_string(self, data, source_path=None, is_two_file=True, post=None, lang=None):
+        """Compile notebooks into HTML strings."""
+        output = self._compile_string(nbformat.reads(data, current_nbformat))
+        return self.site.apply_shortcodes(output, filename=source_path, with_dependencies=True, extra_context={'post': post})
+
     # TODO remove in v8
     def compile_html_string(self, source, is_two_file=True):
         """Export notebooks as HTML strings."""
-        return self.compile_string(source, is_two_file)
+        with io.open(source, "r", encoding="utf8") as in_file:
+            nb_json = nbformat.read(in_file, current_nbformat)
+        return self._compile_string(nb_json)
 
     def compile(self, source, dest, is_two_file=False, post=None, lang=None):
         """Compile the source file into HTML and save as dest."""
         makedirs(os.path.dirname(dest))
         with io.open(dest, "w+", encoding="utf8") as out_file:
-            output = self.compile_string(source, is_two_file)
-            output, shortcode_deps = self.site.apply_shortcodes(output, filename=source, with_dependencies=True, extra_context=dict(post=post))
+            with io.open(source, "r", encoding="utf8") as in_file:
+                nb_str = in_file.read()
+            output, shortcode_deps = self.compile_string(nb_str, is_two_file, post, lang)
             out_file.write(output)
         if post is None:
             if shortcode_deps:

diff --git a/nikola/plugins/compile/markdown/__init__.py b/nikola/plugins/compile/markdown/__init__.py
@@ -63,14 +63,14 @@ def set_site(self, site):
 
         self.config_dependencies.append(str(sorted(site.config.get("MARKDOWN_EXTENSIONS"))))
 
-    def compile_string(self, content, source_path=None, is_two_file=True, post=None):
+    def compile_string(self, data, source_path=None, is_two_file=True, post=None, lang=None):
         """Compile Markdown into HTML strings."""
         if markdown is None:
             req_missing(['markdown'], 'build this site (compile Markdown)')
         self.extensions += self.site.config.get("MARKDOWN_EXTENSIONS")
         if not is_two_file:
-            _, content = self.split_metadata(content)
-        output = markdown(content, self.extensions, output_format="html5")
+            _, data = self.split_metadata(data)
+        output = markdown(data, self.extensions, output_format="html5")
         output, shortcode_deps = self.site.apply_shortcodes(output, filename=source_path, with_dependencies=True, extra_context={'post': post})
         return output, shortcode_deps
 

diff --git a/nikola/plugins/compile/pandoc.py b/nikola/plugins/compile/pandoc.py
@@ -70,6 +70,10 @@ def compile(self, source, dest, is_two_file=True, post=None, lang=None):
             if e.strreror == 'No such file or directory':
                 req_missing(['pandoc'], 'build this site (compile with pandoc)', python=False)
 
+    def compile_string(self, data, source_path=None, is_two_file=True, post=None, lang=None):
+        """Compile into HTML strings."""
+        raise ValueError("Pandoc compiler does not support compile_string due to multiple output formats")
+
     def create_post(self, path, **kw):
         """Create a new post."""
         content = kw.pop('content', None)

diff --git a/nikola/plugins/compile/php.py b/nikola/plugins/compile/php.py
@@ -51,6 +51,10 @@ def compile(self, source, dest, is_two_file=True, post=None, lang=None):
                 out_file.write('<!-- __NIKOLA_PHP_TEMPLATE_INJECTION source:{0} checksum:{1}__ -->'.format(source, hash))
         return True
 
+    def compile_string(self, data, source_path=None, is_two_file=True, post=None, lang=None):
+        """Compile PHP into HTML strings."""
+        return data
+
     def create_post(self, path, **kw):
         """Create a new post."""
         content = kw.pop('content', None)

diff --git a/nikola/plugins/compile/rest/__init__.py b/nikola/plugins/compile/rest/__init__.py
@@ -59,7 +59,7 @@ class CompileRest(PageCompiler):
     demote_headers = True
     logger = None
 
-    def compile_string(self, data, source_path=None, is_two_file=True):
+    def compile_string(self, data, source_path=None, is_two_file=True, post=None, lang=None):
         """Compile reST into HTML strings."""
         # If errors occur, this will be added to the line number reported by
         # docutils so the line number matches the actual line number (off by
@@ -88,7 +88,8 @@ def compile_string(self, data, source_path=None, is_two_file=True):
             # To prevent some weird bugs here or there.
             # Original issue: empty files.  `output` became a bytestring.
             output = output.decode('utf-8')
-        return output, error_level, deps
+        output, shortcode_deps = self.site.apply_shortcodes(output, filename=source_path, with_dependencies=True, extra_context=dict(post=post))
+        return output, error_level, deps, shortcode_deps
 
     # TODO remove in v8
     def compile_html_string(self, data, source_path=None, is_two_file=True):
@@ -102,8 +103,7 @@ def compile(self, source, dest, is_two_file=True, post=None, lang=None):
         with io.open(dest, "w+", encoding="utf8") as out_file:
             with io.open(source, "r", encoding="utf8") as in_file:
                 data = in_file.read()
-                output, error_level, deps = self.compile_string(data, source, is_two_file)
-                output, shortcode_deps = self.site.apply_shortcodes(output, filename=source, with_dependencies=True, extra_context=dict(post=post))
+                output, error_level, deps, shortcode_deps = self.compile_string(data, source, is_two_file)
                 out_file.write(output)
             if post is None:
                 if deps.list:

diff --git a/nikola/plugins/task/listings.py b/nikola/plugins/task/listings.py
@@ -29,6 +29,7 @@
 from __future__ import unicode_literals, print_function
 
 from collections import defaultdict
+import io
 import os
 import lxml.html
 
@@ -116,7 +117,9 @@ def render_listing(in_name, out_name, input_folder, output_folder, folders=[], f
             if in_name and in_name.endswith('.ipynb'):
                 # Special handling: render ipynbs in listings (Issue #1900)
                 ipynb_compiler = self.site.plugin_manager.getPluginByName("ipynb", "PageCompiler").plugin_object
-                ipynb_raw = ipynb_compiler.compile_string(in_name, True)
+                with io.open(in_name, "r", encoding="utf8") as in_file:
+                    nb_json = ipynb_compiler._nbformat_read(in_file)
+                    ipynb_raw = ipynb_compiler._compile_string(nb_json)
                 ipynb_html = lxml.html.fromstring(ipynb_raw)
                 # The raw HTML contains garbage (scripts and styles), we can’t leave it in
                 code = lxml.html.tostring(ipynb_html.xpath('//*[@id="notebook"]')[0], encoding='unicode')