Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Merge pull request #2695 from getnikola/fix-2496-second-try
Attempting to fix #2496.
  • Loading branch information
Kwpolska committed Apr 2, 2017
2 parents e98c1cf + c34d5e8 commit 4fb64ab
Show file tree
Hide file tree
Showing 5 changed files with 92 additions and 60 deletions.
2 changes: 2 additions & 0 deletions CHANGES.txt
Expand Up @@ -10,6 +10,8 @@ Bugfixes
* Read data files only if Nikola configuration exists (Issue #2708)
* Make ``PAGE_INDEX`` work with ``PRETTY_URLS`` (Issue #2705)
* Fix PHP posts/pages not rendering on Windows (Issue #2706)
* Improving support for PostScanner plugins by asking them which compilers
are unused (Issue #2496)

New in v7.8.4
=============
Expand Down
140 changes: 81 additions & 59 deletions nikola/nikola.py
Expand Up @@ -960,31 +960,9 @@ def __init__(self, **config):
for k, v in self.config['COMPILERS'].items():
self.config['_COMPILERS_RAW'][k] = list(v)

compilers = defaultdict(set)
# Also add aliases for combinations with TRANSLATIONS_PATTERN
for compiler, exts in self.config['COMPILERS'].items():
for ext in exts:
compilers[compiler].add(ext)
for lang in self.config['TRANSLATIONS'].keys():
candidate = utils.get_translation_candidate(self.config, "f" + ext, lang)
compilers[compiler].add(candidate)

# Get search path for themes
self.themes_dirs = ['themes'] + self.config['EXTRA_THEMES_DIRS']

# Avoid redundant compilers
# Remove compilers that match nothing in POSTS/PAGES
# And put them in "bad compilers"
pp_exts = set([os.path.splitext(x[0])[1] for x in self.config['post_pages']])
self.config['COMPILERS'] = {}
self.disabled_compilers = {}
self.bad_compilers = set([])
for k, v in compilers.items():
if pp_exts.intersection(v):
self.config['COMPILERS'][k] = sorted(list(v))
else:
self.bad_compilers.add(k)

self._set_global_context_from_config()
# Read data files only if a site exists (Issue #2708)
if self.configured:
Expand All @@ -1001,6 +979,28 @@ def __init__(self, **config):
self.state._set_site(self)
self.cache._set_site(self)

def _filter_duplicate_plugins(self, plugin_list):
"""Find repeated plugins and discard the less local copy."""
def plugin_position_in_places(plugin):
# plugin here is a tuple:
# (path to the .plugin file, path to plugin module w/o .py, plugin metadata)
for i, place in enumerate(self._plugin_places):
if plugin[0].startswith(place):
return i

plugin_dict = defaultdict(list)
for data in plugin_list:
plugin_dict[data[2].name].append(data)
result = []
for _, plugins in plugin_dict.items():
if len(plugins) > 1:
# Sort by locality
plugins.sort(key=plugin_position_in_places)
utils.LOGGER.debug("Plugin {} exists in multiple places, using {}".format(
plugins[-1][2].name, plugins[-1][0]))
result.append(plugins[-1])
return result

def init_plugins(self, commands_only=False, load_all=False):
"""Load plugins as needed."""
self.plugin_manager = PluginManager(categories_filter={
Expand Down Expand Up @@ -1034,6 +1034,23 @@ def init_plugins(self, commands_only=False, load_all=False):
os.path.expanduser('~/.nikola/plugins'),
] + [utils.sys_encode(path) for path in extra_plugins_dirs if path]

compilers = defaultdict(set)
# Also add aliases for combinations with TRANSLATIONS_PATTERN
for compiler, exts in self.config['COMPILERS'].items():
for ext in exts:
compilers[compiler].add(ext)
for lang in self.config['TRANSLATIONS'].keys():
candidate = utils.get_translation_candidate(self.config, "f" + ext, lang)
compilers[compiler].add(candidate)

# Avoid redundant compilers (if load_all is False):
# Remove compilers (and corresponding compiler extensions) that are not marked as
# needed by any PostScanner plugin and put them into self.disabled_compilers
# (respectively self.disabled_compiler_extensions).
self.config['COMPILERS'] = {}
self.disabled_compilers = {}
self.disabled_compiler_extensions = defaultdict(list)

self.plugin_manager.getPluginLocator().setPluginPlaces(self._plugin_places)
self.plugin_manager.locatePlugins()
bad_candidates = set([])
Expand All @@ -1047,47 +1064,58 @@ def init_plugins(self, commands_only=False, load_all=False):
else:
bad_candidates.add(p)
elif self.configured: # Not commands-only, and configured
# Remove compilers we don't use
if p[-1].name in self.bad_compilers:
bad_candidates.add(p)
self.disabled_compilers[p[-1].name] = p
utils.LOGGER.debug('Not loading unneeded compiler {}', p[-1].name)
if p[-1].name not in self.config['COMPILERS'] and \
p[-1].details.has_option('Nikola', 'PluginCategory') and p[-1].details.get('Nikola', 'PluginCategory') in ('Compiler', 'PageCompiler'):
bad_candidates.add(p)
self.disabled_compilers[p[-1].name] = p
utils.LOGGER.debug('Not loading unneeded compiler {}', p[-1].name)
# Remove blacklisted plugins
if p[-1].name in self.config['DISABLED_PLUGINS']:
bad_candidates.add(p)
utils.LOGGER.debug('Not loading disabled plugin {}', p[-1].name)
# Remove compilers we don't use
if p[-1].details.has_option('Nikola', 'PluginCategory') and p[-1].details.get('Nikola', 'PluginCategory') in ('Compiler', 'PageCompiler'):
bad_candidates.add(p)
self.disabled_compilers[p[-1].name] = p
# Remove compiler extensions we don't need
if p[-1].details.has_option('Nikola', 'compiler') and p[-1].details.get('Nikola', 'compiler') in self.disabled_compilers:
bad_candidates.add(p)
utils.LOGGER.debug('Not loading compiler extension {}', p[-1].name)
self.disabled_compiler_extensions[p[-1].details.get('Nikola', 'compiler')].append(p)
self.plugin_manager._candidates = list(set(self.plugin_manager._candidates) - bad_candidates)

# Find repeated plugins and discard the less local copy
def plugin_position_in_places(plugin):
# plugin here is a tuple:
# (path to the .plugin file, path to plugin module w/o .py, plugin metadata)
for i, place in enumerate(self._plugin_places):
if plugin[0].startswith(place):
return i
self.plugin_manager._candidates = self._filter_duplicate_plugins(self.plugin_manager._candidates)
self.plugin_manager.loadPlugins()

plugin_dict = defaultdict(list)
for data in self.plugin_manager._candidates:
plugin_dict[data[2].name].append(data)
self.plugin_manager._candidates = []
for name, plugins in plugin_dict.items():
if len(plugins) > 1:
# Sort by locality
plugins.sort(key=plugin_position_in_places)
utils.LOGGER.debug("Plugin {} exists in multiple places, using {}".format(
plugins[-1][2].name, plugins[-1][0]))
self.plugin_manager._candidates.append(plugins[-1])
# Search for compiler plugins which we disabled but shouldn't have
self._activate_plugins_of_category("PostScanner")
if not load_all:
file_extensions = set()
for post_scanner in [p.plugin_object for p in self.plugin_manager.getPluginsOfCategory('PostScanner')]:
exts = post_scanner.supported_extensions()
if exts is not None:
file_extensions.update(exts)
else:
# Stop scanning for more: once we get None, we have to load all compilers anyway
utils.LOGGER.debug("Post scanner {0!r} does not implement `supported_extensions`, loading all compilers".format(post_scanner))
file_extensions = None
break
to_add = []
for k, v in compilers.items():
if file_extensions is None or file_extensions.intersection(v):
self.config['COMPILERS'][k] = sorted(list(v))
p = self.disabled_compilers.pop(k, None)
if p:
to_add.append(p)
for p in self.disabled_compiler_extensions.pop(k, []):
to_add.append(p)
for _, p in self.disabled_compilers.items():
utils.LOGGER.debug('Not loading unneeded compiler {}', p[-1].name)
for _, plugins in self.disabled_compiler_extensions.items():
for p in plugins:
utils.LOGGER.debug('Not loading compiler extension {}', p[-1].name)
if to_add:
self.plugin_manager._candidates = self._filter_duplicate_plugins(to_add)
self.plugin_manager.loadPlugins()

self.plugin_manager.loadPlugins()
# IPython theme configuration. If a website has ipynb enabled in post_pages
# we should enable the IPython CSS (leaving that up to the theme itself).
if 'needs_ipython_css' not in self._GLOBAL_CONTEXT:
self._GLOBAL_CONTEXT['needs_ipython_css'] = 'ipynb' in self.config['COMPILERS']

self._activate_plugins_of_category("Taxonomy")
self.taxonomy_plugins = {}
Expand All @@ -1111,7 +1139,6 @@ def plugin_position_in_places(plugin):
plugin_info.plugin_object.short_help = plugin_info.description
self._commands[plugin_info.name] = plugin_info.plugin_object

self._activate_plugins_of_category("PostScanner")
self._activate_plugins_of_category("Task")
self._activate_plugins_of_category("LateTask")
self._activate_plugins_of_category("TaskMultiplier")
Expand Down Expand Up @@ -1231,11 +1258,6 @@ def _set_global_context_from_config(self):
self._GLOBAL_CONTEXT['sort_posts'] = utils.sort_posts
self._GLOBAL_CONTEXT['meta_generator_tag'] = self.config.get('META_GENERATOR_TAG')

# IPython theme configuration. If a website has ipynb enabled in post_pages
# we should enable the IPython CSS (leaving that up to the theme itself).

self._GLOBAL_CONTEXT['needs_ipython_css'] = 'ipynb' in self.config['COMPILERS']

self._GLOBAL_CONTEXT.update(self.config.get('GLOBAL_CONTEXT', {}))

def _set_global_context_from_data(self):
Expand Down Expand Up @@ -2402,7 +2424,7 @@ def atom_post_text(post, text):
entry_title = lxml.etree.SubElement(entry_root, "title")
entry_title.text = post.title(lang)
entry_id = lxml.etree.SubElement(entry_root, "id")
entry_id.text = post.guid(lang)
entry_id.text = post.permalink(lang, absolute=True)
entry_updated = lxml.etree.SubElement(entry_root, "updated")
entry_updated.text = post.formatted_updated('webiso')
entry_published = lxml.etree.SubElement(entry_root, "published")
Expand Down
4 changes: 4 additions & 0 deletions nikola/plugin_categories.py
Expand Up @@ -95,6 +95,10 @@ def scan(self):
"""Create a list of posts from some source. Returns a list of Post objects."""
raise NotImplementedError()

def supported_extensions(self):
"""Return a list of supported file extensions, or None if such a list isn't known beforehand."""
return None


class Command(BasePlugin, DoitCommand):
"""Doit command implementation."""
Expand Down
2 changes: 1 addition & 1 deletion nikola/plugins/command/new_post.py
Expand Up @@ -544,6 +544,6 @@ def print_compilers(self):
More compilers are available in the Plugins Index.
Compilers marked with ! and ~ require additional configuration:
! not in the PAGES/POSTS tuples (unused)
! not in the POSTS/PAGES tuples and any post scanners (unused)
~ not in the COMPILERS dict (disabled)
Read more: {0}""".format(COMPILERS_DOC_LINK))
4 changes: 4 additions & 0 deletions nikola/plugins/misc/scan_posts.py
Expand Up @@ -106,3 +106,7 @@ def scan(self):
raise err

return timeline

def supported_extensions(self):
"""Return a list of supported file extensions, or None if such a list isn't known beforehand."""
return list({os.path.splitext(x[0])[1] for x in self.site.config['post_pages']})

0 comments on commit 4fb64ab

Please sign in to comment.