Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Merge pull request #2443 from getnikola/return-dependencies-from-shor…
…tcodes

Initial implementantion of deps for shortcodes
  • Loading branch information
ralsina committed Aug 16, 2016
2 parents 880a50a + c76b260 commit 5e5cfa2
Show file tree
Hide file tree
Showing 13 changed files with 110 additions and 40 deletions.
14 changes: 13 additions & 1 deletion docs/extending.txt
Expand Up @@ -577,14 +577,26 @@ So, if you are creating a plugin that generates markup, it may be a good idea
to register it as a shortcode in addition of to restructured text directive or
markdown extension, thus making it available to all markup formats.

To implement your own shortcodes from a plugin, you can create a plugin inheriting ``ShortcodePlugin`` and call
To implement your own shortcodes from a plugin, you can create a plugin inheriting ``ShortcodePlugin`` and
from its ``set_site`` method, call

``Nikola.register_shortcode(name, func)`` with the following arguments:

``name``:
Name of the shortcode ("foo" in the examples above)
``func``:
A function that will handle the shortcode

The shortcode handler **must** return a two-element tuple, ``(output, dependencies)``

``output``:
The text that will replace the shortcode in the document.

``dependencies``:
A list of all the files on disk which will make the output be considered
out of date. For example, if the shortcode uses a template, it should be
the path to the template file.

The shortcode handler **must** accept the following named arguments (or
variable keyword arguments):

Expand Down
18 changes: 12 additions & 6 deletions nikola/nikola.py
Expand Up @@ -1426,7 +1426,7 @@ def url_replacer(self, src, dst, lang=None, url_type=None):

return result

def _make_renderfunc(self, t_data):
def _make_renderfunc(self, t_data, fname=None):
"""Return a function that can be registered as a template shortcode.
The returned function has access to the passed template data and
Expand All @@ -1443,7 +1443,12 @@ def render_shortcode(*args, **kw):
context['lang'] = utils.LocaleBorg().current_lang
for k in self._GLOBAL_CONTEXT_TRANSLATABLE:
context[k] = context[k](context['lang'])
return self.template_system.render_template_to_string(t_data, context)
output = self.template_system.render_template_to_string(t_data, context)
if fname is not None:
dependencies = [fname] + self.template_system.get_deps(fname)
else:
dependencies = []
return output, dependencies
return render_shortcode

def _register_templated_shortcodes(self):
Expand All @@ -1461,9 +1466,9 @@ def _register_templated_shortcodes(self):

if ext != '.tmpl':
continue

with open(os.path.join(sc_dir, fname)) as fd:
self.register_shortcode(name, self._make_renderfunc(fd.read()))
self.register_shortcode(name, self._make_renderfunc(
fd.read(), os.path.join(sc_dir, fname)))

def register_shortcode(self, name, f):
"""Register function f to handle shortcode "name"."""
Expand All @@ -1472,11 +1477,12 @@ def register_shortcode(self, name, f):
return
self.shortcode_registry[name] = f

def apply_shortcodes(self, data, filename=None, lang=None):
# XXX in v8, get rid of with_dependencies
def apply_shortcodes(self, data, filename=None, lang=None, with_dependencies=False):
"""Apply shortcodes from the registry on data."""
if lang is None:
lang = utils.LocaleBorg().current_lang
return shortcodes.apply_shortcodes(data, self.shortcode_registry, self, filename, lang=lang)
return shortcodes.apply_shortcodes(data, self.shortcode_registry, self, filename, lang=lang, with_dependencies=with_dependencies)

def generic_rss_renderer(self, lang, title, link, description, timeline, output_path,
rss_teasers, rss_plain, feed_length=10, feed_url=None,
Expand Down
4 changes: 4 additions & 0 deletions nikola/plugin_categories.py
Expand Up @@ -82,6 +82,10 @@ def inject_dependency(self, target, dependency):
"""Add 'dependency' to the target task's task_deps."""
self.site.injected_deps[target].append(dependency)

def get_deps(self, filename):
"""Find the dependencies for a file."""
return []


class PostScanner(BasePlugin):
"""The scan method of these plugins is called by Nikola.scan_posts."""
Expand Down
11 changes: 10 additions & 1 deletion nikola/plugins/compile/html.py
Expand Up @@ -49,8 +49,17 @@ def compile_html(self, source, dest, is_two_file=True):
data = in_file.read()
if not is_two_file:
_, data = self.split_metadata(data)
data = self.site.apply_shortcodes(source)
data, shortcode_deps = self.site.apply_shortcodes(source, with_dependencies=True)
out_file.write(data)
try:
post = self.site.post_per_input_file[source]
except KeyError:
if shortcode_deps:
self.logger.error(
"Cannot save dependencies for post {0} due to unregistered source file name",
source)
else:
post._depfile[dest] += shortcode_deps
return True

def create_post(self, path, **kw):
Expand Down
11 changes: 10 additions & 1 deletion nikola/plugins/compile/ipynb.py
Expand Up @@ -93,8 +93,17 @@ def compile_html(self, source, dest, is_two_file=True):
makedirs(os.path.dirname(dest))
with io.open(dest, "w+", encoding="utf8") as out_file:
output = self.compile_html_string(source, is_two_file)
output = self.site.apply_shortcodes(output, filename=source)
output, shortcode_deps = self.site.apply_shortcodes(output, filename=source, with_dependencies=True)
out_file.write(output)
try:
post = self.site.post_per_input_file[source]
except KeyError:
if shortcode_deps:
self.logger.error(
"Cannot save dependencies for post {0} due to unregistered source file name",
source)
else:
post._depfile[dest] += shortcode_deps

def read_metadata(self, post, file_metadata_regexp=None, unslugify_titles=False, lang=None):
"""Read metadata directly from ipynb file.
Expand Down
11 changes: 10 additions & 1 deletion nikola/plugins/compile/markdown/__init__.py
Expand Up @@ -75,8 +75,17 @@ def compile_html(self, source, dest, is_two_file=True):
if not is_two_file:
_, data = self.split_metadata(data)
output = markdown(data, self.extensions)
output = self.site.apply_shortcodes(output, filename=source)
output, shortcode_deps = self.site.apply_shortcodes(output, filename=source, with_dependencies=True)
out_file.write(output)
try:
post = self.site.post_per_input_file[source]
except KeyError:
if shortcode_deps:
self.logger.error(
"Cannot save dependencies for post {0} due to unregistered source file name",
source)
else:
post._depfile[dest] += shortcode_deps

def create_post(self, path, **kw):
"""Create a new post."""
Expand Down
11 changes: 10 additions & 1 deletion nikola/plugins/compile/pandoc.py
Expand Up @@ -56,9 +56,18 @@ def compile_html(self, source, dest, is_two_file=True):
try:
subprocess.check_call(['pandoc', '-o', dest, source] + self.site.config['PANDOC_OPTIONS'])
with open(dest, 'r', encoding='utf-8') as inf:
output = self.site.apply_shortcodes(inf.read())
output, shortcode_deps = self.site.apply_shortcodes(inf.read(), with_dependencies=True)
with open(dest, 'w', encoding='utf-8') as outf:
outf.write(output)
try:
post = self.site.post_per_input_file[source]
except KeyError:
if shortcode_deps:
self.logger.error(
"Cannot save dependencies for post {0} due to unregistered source file name",
source)
else:
post._depfile[dest] += shortcode_deps
except OSError as e:
if e.strreror == 'No such file or directory':
req_missing(['pandoc'], 'build this site (compile with pandoc)', python=False)
Expand Down
3 changes: 2 additions & 1 deletion nikola/plugins/compile/rest/__init__.py
Expand Up @@ -98,7 +98,7 @@ def compile_html(self, source, dest, is_two_file=True):
with io.open(source, "r", encoding="utf8") as in_file:
data = in_file.read()
output, error_level, deps = self.compile_html_string(data, source, is_two_file)
output = self.site.apply_shortcodes(output, filename=source)
output, shortcode_deps = self.site.apply_shortcodes(output, filename=source, with_dependencies=True)
out_file.write(output)
try:
post = self.site.post_per_input_file[source]
Expand All @@ -109,6 +109,7 @@ def compile_html(self, source, dest, is_two_file=True):
source)
else:
post._depfile[dest] += deps.list
post._depfile[dest] += shortcode_deps
if error_level < 3:
return True
else:
Expand Down
37 changes: 18 additions & 19 deletions nikola/plugins/template/jinja.py
Expand Up @@ -31,7 +31,6 @@
import os
import io
import json
from collections import deque
try:
import jinja2
from jinja2 import meta
Expand All @@ -48,6 +47,7 @@ class JinjaTemplates(TemplateSystem):
name = "jinja"
lookup = None
dependency_cache = {}
per_file_cache = {}

def __init__(self):
"""Initialize Jinja2 environment with extended set of filters."""
Expand Down Expand Up @@ -103,27 +103,26 @@ def render_template_to_string(self, template, context):
"""Render template to a string using context."""
return self.lookup.from_string(template).render(**context)

def get_deps(self, filename):
"""Return paths to dependencies for the template loaded from filename."""
deps = set([])
with open(filename) as fd:
source = fd.read()
ast = self.lookup.parse(source)
dep_names = meta.find_referenced_templates(ast)
for dep_name in dep_names:
filename = self.lookup.loader.get_source(self.lookup, dep_name)[1]
deps.add(filename)
sub_deps = self.get_deps(filename)
self.dependency_cache[dep_name] = sub_deps
deps |= set(sub_deps)
return list(deps)

def template_deps(self, template_name):
"""Generate list of dependencies for a template."""
# Cache the lists of dependencies for each template name.
if self.dependency_cache.get(template_name) is None:
# Use a breadth-first search to find all templates this one
# depends on.
queue = deque([template_name])
visited_templates = set([template_name])
deps = []
while len(queue) > 0:
curr = queue.popleft()
source, filename = self.lookup.loader.get_source(self.lookup,
curr)[:2]
deps.append(filename)
ast = self.lookup.parse(source)
dep_names = meta.find_referenced_templates(ast)
for dep_name in dep_names:
if (dep_name not in visited_templates and dep_name is not None):
visited_templates.add(dep_name)
queue.append(dep_name)
self.dependency_cache[template_name] = deps
filename = self.lookup.loader.get_source(self.lookup, template_name)[1]
self.dependency_cache[template_name] = [filename] + self.get_deps(filename)
return self.dependency_cache[template_name]

def get_template_path(self, template_name):
Expand Down
13 changes: 9 additions & 4 deletions nikola/plugins/template/mako.py
Expand Up @@ -56,7 +56,7 @@ class MakoTemplates(TemplateSystem):
cache_dir = None

def get_deps(self, filename):
"""Get dependencies for a template (internal function)."""
"""Get paths to dependencies for a template."""
text = util.read_file(filename)
lex = lexer.Lexer(text=text, filename=filename)
lex.parse()
Expand All @@ -66,6 +66,11 @@ def get_deps(self, filename):
keyword = getattr(n, 'keyword', None)
if keyword in ["inherit", "namespace"] or isinstance(n, parsetree.IncludeTag):
deps.append(n.attributes['file'])
# Some templates will include "foo.tmpl" and we need paths, so normalize them
# using the template lookup
for i, d in enumerate(deps):
if os.sep not in d:
deps[i] = self.get_template_path(d)
return deps

def set_directories(self, directories, cache_folder):
Expand Down Expand Up @@ -116,7 +121,7 @@ def render_template(self, template_name, output_name, context):
def render_template_to_string(self, template, context):
"""Render template to a string using context."""
context.update(self.filters)
return Template(template).render(**context)
return Template(template, lookup=self.lookup).render(**context)

def template_deps(self, template_name):
"""Generate list of dependencies for a template."""
Expand All @@ -127,8 +132,8 @@ def template_deps(self, template_name):
dep_filenames = self.get_deps(template.filename)
deps = [template.filename]
for fname in dep_filenames:
deps += self.template_deps(fname)
self.cache[template_name] = tuple(deps)
deps += self.get_deps(fname)
self.cache[template_name] = deps
return list(self.cache[template_name])

def get_template_path(self, template_name):
Expand Down
13 changes: 10 additions & 3 deletions nikola/shortcodes.py
Expand Up @@ -256,7 +256,8 @@ def _split_shortcodes(data):
return result


def apply_shortcodes(data, registry, site=None, filename=None, raise_exceptions=False, lang=None):
# FIXME: in v8, get rid of with_dependencies
def apply_shortcodes(data, registry, site=None, filename=None, raise_exceptions=False, lang=None, with_dependencies=False):
"""Apply Hugo-style shortcodes on data.
{{% name parameters %}} will end up calling the registered "name" function with the given parameters.
Expand All @@ -279,6 +280,7 @@ def apply_shortcodes(data, registry, site=None, filename=None, raise_exceptions=
sc_data = _split_shortcodes(data)
# Now process data
result = []
dependencies = []
pos = 0
while pos < len(sc_data):
current = sc_data[pos]
Expand Down Expand Up @@ -315,10 +317,15 @@ def apply_shortcodes(data, registry, site=None, filename=None, raise_exceptions=
if getattr(f, 'nikola_shortcode_pass_filename', None):
kw['filename'] = filename
res = f(*args, **kw)
if not isinstance(res, tuple): # For backards compatibility
res = (res, [])
else:
LOGGER.error('Unknown shortcode {0} (started at {1})', name, _format_position(data, current[2]))
res = ''
result.append(res)
res = ('', [])
result.append(res[0])
dependencies += res[1]
if with_dependencies:
return empty_string.join(result), dependencies
return empty_string.join(result)
except ParsingError as e:
if raise_exceptions:
Expand Down
2 changes: 1 addition & 1 deletion tests/base.py
Expand Up @@ -229,4 +229,4 @@ def register_shortcode(self, name, f):

def apply_shortcodes(self, data, *a, **kw):
"""Apply shortcodes from the registry on data."""
return nikola.shortcodes.apply_shortcodes(data, self.shortcode_registry)
return nikola.shortcodes.apply_shortcodes(data, self.shortcode_registry, **kw)
2 changes: 1 addition & 1 deletion tests/test_rst_compiler.py
Expand Up @@ -4,7 +4,7 @@


""" Test cases for Nikola ReST extensions.
A base class ReSTExtensionTestCase provides the tests basic behaivor.
A base class ReSTExtensionTestCase provides the tests basic behaviour.
Subclasses must override the "sample" class attribute with the ReST markup.
The sample will be rendered as HTML using publish_parts() by setUp().
One method is provided for checking the resulting HTML:
Expand Down

0 comments on commit 5e5cfa2

Please sign in to comment.