Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Adding taxonomy plugin type and basic handling.
  • Loading branch information
felixfontein committed Oct 16, 2016
1 parent f5d42cf commit 85fac3d
Show file tree
Hide file tree
Showing 6 changed files with 587 additions and 0 deletions.
3 changes: 3 additions & 0 deletions nikola/nikola.py
Expand Up @@ -73,6 +73,7 @@
SignalHandler,
ConfigPlugin,
PostScanner,
Taxonomy,
)

if DEBUG:
Expand Down Expand Up @@ -948,6 +949,7 @@ def init_plugins(self, commands_only=False, load_all=False):
"SignalHandler": SignalHandler,
"ConfigPlugin": ConfigPlugin,
"PostScanner": PostScanner,
"Taxonomy": Taxonomy,
})
self.plugin_manager.getPluginLocator().setPluginInfoExtension('plugin')
extra_plugins_dirs = self.config['EXTRA_PLUGINS_DIRS']
Expand Down Expand Up @@ -1019,6 +1021,7 @@ def plugin_position_in_places(plugin):

self.plugin_manager.loadPlugins()

self._activate_plugins_of_category("Taxonomy")
self._activate_plugins_of_category("SignalHandler")

# Emit signal for SignalHandlers which need to start running immediately.
Expand Down
123 changes: 123 additions & 0 deletions nikola/plugin_categories.py
Expand Up @@ -49,6 +49,7 @@
'SignalHandler',
'ConfigPlugin',
'PostScanner',
'Taxonomy',
)


Expand Down Expand Up @@ -451,3 +452,125 @@ def import_file(self):
def save_post(self):
"""Save a post to disk."""
raise NotImplementedError()


class Taxonomy(BasePlugin):
"""Taxonomy for posts.
A taxonomy plugin allows to classify posts (see #2107) by
classification strings.
"""

name = "dummy_taxonomy"

# Adjust the following values in your plugin!

# The classification name to be used for path handlers.
classification_name = "taxonomy"
# The classification name to be used when storing the classification
# in the metadata.
metadata_name = "taxonomy"
# If True, there can be more than one classification per post; in that case,
# the classification data in the metadata is stored as a list. If False,
# the classification data in the metadata is stored as a string, or None
# when no classification is given.
more_than_one_classifications_per_post = False
# Whether the classification has a hierarchy.
has_hierarchy = False
# If True, the list for a classification includes all posts with a
# sub-classification (in case has_hierarchy is True).
include_posts_from_subhierarchies = False
# Whether to show the posts for one classification as an index or
# as a post list.
show_list_as_index = False
# The template to use for the post list for one classification.
# Set to none to avoid generating overviews.
template_for_list_of_one_classification = "tagindex.tmpl"
# The template to use for the classification overview page.
template_for_classification_overview = "list.tmpl"
# Whether this classification applies to posts.
apply_to_posts = True
# Whether this classification applies to pages.
apply_to_pages = False
# The minimum number of posts a classification must have to be listed in
# the overview.
minimum_post_count_per_classification_in_overview = 1
# Whether post lists resp. indexes should be created for empty
# classifications.
omit_empty_classifications = False
# Whether to include all classifications for all languages in every
# language, or only the classifications for one language in its language's
# pages.
also_create_classifications_from_other_languages = True

def classify(self, post, lang):
"""Classifies the given post for the given language.
Must return a list or tuple of strings."""
raise NotImplementedError()

def sort_posts(self, posts):
"""Sorts the given list of posts."""
pass

def get_list_path(self, lang):
"""A path handler for the list of all classifications.
The last element in the returned path must have no extension, and the
PRETTY_URLS config must be ignored. The return value will be modified
based on the PRETTY_URLS and INDEX_FILE settings."""
raise NotImplementedError()

def get_path(self, classification, lang):
"""A path handler for the given classification.
The last element in the returned path must have no extension, and the
PRETTY_URLS config must be ignored. The return value will be modified
based on the PRETTY_URLS and INDEX_FILE settings.
For hierarchical taxonomies, the result of extract_hierarchy is provided.
For non-hierarchical taxonomies, the classification string itself is provided."""
raise NotImplementedError()

def extract_hierarchy(self, classification):
"""Given a classification, return a list of parts in the hierarchy.
For non-hierarchical taxonomies, it usually suffices to return
`[classification]`."""
return [classification]

def recombine_classification_from_hierarchy(self, hierarchy):
"""Given a list of parts in the hierarchy, return the classification string.
For non-hierarchical taxonomies, it usually suffices to return hierarchy[0]."""
return hierarchy[0]

def provide_list_context_and_uptodate(self):
"""Provides data for the context and the uptodate list for the list of all classifiations.
Must return a tuple of two dicts. The first is merged into the page's context,
the second will be put into the uptodate list of all generated tasks.
Context must contain `title`."""
raise NotImplementedError()

def provide_context_and_uptodate(self, classification):
"""Provides data for the context and the uptodate list for the list of the given classifiation.
Must return a tuple of two dicts. The first is merged into the page's context,
the second will be put into the uptodate list of all generated tasks.
Context must contain `title`, which should be something like 'Posts about <classification>',
and `classification_title`, which should be related to the classification string."""
raise NotImplementedError()

def postprocess_posts_per_classification(self, posts_per_classification_per_language, flat_hierarchy_per_lang=None, hierarchy_lookup_per_lang=None):
"""This function can rearrange, modify or otherwise use the list of posts per classification and per language.
For compatibility reasons, the list could be stored somewhere else as well.
In case `has_hierarchy` is `True`, `flat_hierarchy_per_lang` is the flat
hierarchy consisting of `TreeNode` elements, and `hierarchy_lookup_per_lang`
is the corresponding hierarchy lookup mapping classification strings to
`TreeNode` objects."""
pass
12 changes: 12 additions & 0 deletions nikola/plugins/misc/taxonomies_classifier.plugin
@@ -0,0 +1,12 @@
[Core]
name = classify_taxonomies
module = classify_taxonomies

[Documentation]
author = Roberto Alsina
version = 1.0
website = https://getnikola.com/
description = Classifies the timeline into taxonomies.

[Nikola]
plugincategory = SignalHandler
210 changes: 210 additions & 0 deletions nikola/plugins/misc/taxonomies_classifier.py
@@ -0,0 +1,210 @@
# -*- coding: utf-8 -*-

# Copyright © 2012-2016 Roberto Alsina and others.

# Permission is hereby granted, free of charge, to any
# person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the
# Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the
# Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice
# shall be included in all copies or substantial portions of
# the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

"""Render the taxonomy overviews, classification pages and feeds."""

from __future__ import unicode_literals
import blinker
import natsort
import os
import sys

from collections import defaultdict

from nikola.plugin_categories import SignalHandler
from nikola import utils


class TaxonomiesClassifier(SignalHandler):
"""Render the tag/category pages and feeds."""

name = "render_taxonomies"

def _do_classification(self):
taxonomies = self.site.plugin_manager.getPluginsOfCategory('Taxonomy')
self.site.posts_per_classification = {}
for taxonomy in taxonomies:
if taxonomy.classification_name in self.site.posts_per_classification:
raise Exception("Found more than one taxonomy with classification name '{}'!".format(taxonomy.classification_name))
self.site.posts_per_classification[taxonomy.classification_name] = {
lang: defaultdict(set) for lang in self.config['TRANSLATIONS'].keys()
}

# Classify posts
for post in self.timeline:
for taxonomy in taxonomies:
if taxonomy.apply_to_posts if post.is_post else taxonomy.apply_to_pages:
classifications = {}
for lang in self.config['TRANSLATIONS'].keys():
# Extract classifications for this language
classifications[lang] = taxonomy.classify(post, lang)
assert taxonomy.more_than_one_classifications_per_post or len(classifications[lang]) <= 1
# Store in metadata
if taxonomy.more_than_one_classifications_per_post:
post.meta[lang][taxonomy.metadata_name] = classifications[lang]
else:
post.meta[lang][taxonomy.metadata_name] = classifications[lang][0] if len(classifications[lang]) > 0 else None
# Add post to sets
for classification in classifications[lang]:
while classification:
self.site.posts_per_classification[taxonomy.classification_name][lang][classification].add(post)
if not taxonomy.include_posts_from_subhierarchies or not taxonomy.has_hierarchy:
break
classification = taxonomy.recombine_classification_from_hierarchy(taxonomy.extract_hierarchy(classification)[:-1])

# Check for valid paths and for collisions
taxonomy_outputs = {lang: dict() for lang in self.config['TRANSLATIONS'].keys()}
quit = False
for taxonomy in taxonomies:
# Check for collisions (per language)
for lang in self.config['TRANSLATIONS'].keys():
for tlang in self.config['TRANSLATIONS'].keys():
if lang != tlang and not taxonomy.also_create_classifications_from_other_languages:
continue
for classification, posts in self.site.posts_per_classification[taxonomy.classification_name][tlang].items():
# Obtain path as tuple
if taxonomy.has_hierarchy:
path = taxonomy.get_path(taxonomy.extract_hierarchy(classification), lang)
else:
path = taxonomy.get_path(classification, lang)
path = tuple(path)
# Check that path is OK
for path_element in path:
if len(path_element) == 0:
utils.LOGGER.error("{0} {1} yields invalid path '{2}'!".format(taxonomy.classification_name.title(), classification, '/'.join(path)))
quit = True
# Determine collisions
if path in taxonomy_outputs[lang]:
other_classification_name, other_classification, other_posts = taxonomy_outputs[lang][path]
utils.LOGGER.error('You have classifications that are too similar: {0} "{1}" and {1} "{2}"'.format(
taxonomy.classification_name, classification, other_classification_name, other_classification))
utils.LOGGER.error('{0} {1} is used in: {1}'.format(
taxonomy.classification_name.title(), classification, ', '.join(sorted([p.source_path for p in posts]))))
utils.LOGGER.error('{0} {1} is used in: {1}'.format(
other_classification_name.title(), other_classification, ', '.join(sorted([p.source_path for p in other_posts]))))
quit = True
else:
taxonomy_outputs[lang][path] = (taxonomy.classification_name, classification, posts)
if quit:
sys.exit(1)

# Sort everything.
self.site.flat_hierarchy_per_classification = {}
self.site.hierarchy_lookup_per_classification = {}
for taxonomy in taxonomies:
# Sort post lists
for lang, posts_per_classification in self.site.posts_per_classification[taxonomy.classification_name].items():
# Convert sets to lists and sort them
for classification in list(posts_per_classification.keys()):
posts = list(posts_per_classification[classification])
posts.sort(key=lambda p:
(int(p.meta('priority')) if p.meta('priority') else 0,
p.date, p.source_path))
posts.reverse()
posts_per_classification[classification] = posts
# Create hierarchy information
if taxonomy.has_hierarchy:
self.site.flat_hierarchy_per_classification[taxonomy.classification_name] = {}
self.site.hierarchy_lookup_per_classification[taxonomy.classification_name] = {}
for lang, posts_per_classification in self.site.posts_per_classification[taxonomy.classification_name].items():
# Compose hierarchy
hierarchy = {}
for classification in posts_per_classification.keys():
hier = taxonomy.extract_hierarchy(classification)
node = hierarchy
for he in hier:
if he not in node:
node[he] = {}
node = node[he]
hierarchy_lookup = {}

def create_hierarchy(cat_hierarchy, parent=None):
"""Create category hierarchy."""
result = []
for name, children in cat_hierarchy.items():
node = utils.TreeNode(name, parent)
node.children = create_hierarchy(children, node)
node.classification_path = [pn.name for pn in node.get_path()]
node.classification_name = taxonomy.recombine_classification_from_hierarchy(node.classification_path)
hierarchy_lookup[node.classification_name] = node
return natsort.natsorted(result, key=lambda e: e.name, alg=natsort.ns.F | natsort.ns.IC)

root_list = create_hierarchy(hierarchy)
flat_hierarchy = utils.flatten_tree_structure(root_list)
# Store result
self.site.flat_hierarchy_per_classification[taxonomy.classification_name][lang] = flat_hierarchy
self.site.hierarchy_lookup_per_classification[taxonomy.classification_name][lang] = hierarchy_lookup
taxonomy.postprocess_posts_per_classification(self.site.posts_per_classification[taxonomy.classification_name],
self.site.flat_hierarchy_per_classification[taxonomy.classification_name],
self.site.hierarchy_lookup_per_classification[taxonomy.classification_name])
else:
taxonomy.postprocess_posts_per_classification(self.site.posts_per_classification[taxonomy.classification_name], flat_hierarchy, hierarchy_lookup)

def _postprocess_path(self, path, lang, force_extension=None):
if force_extension is not None:
if len(path) == 0:
path = [os.path.splitext(self.site.config['INDEX_FILE'])[0]]
path[-1] += force_extension
elif self.site.config['PRETTY_URLS'] or len(path) == 0:
path = path + [self.site.config['INDEX_FILE']]
else:
path[-1] += '.html'
return [_f for _f in [self.site.config['TRANSLATIONS'][lang]] + path if _f]

def _taxonomy_index_path(self, lang, taxonomy):
"""Return path to the classification overview."""
return self._postprocess_path(taxonomy.get_list_path(lang), lang)

def _taxonomy_path(self, name, lang, taxonomy, force_extension=None):
"""Return path to a classification."""
if taxonomy.has_hirearchy:
path = taxonomy.get_path(taxonomy.extract_hierarchy(name), lang)
else:
path = taxonomy.get_path(name, lang)
return self._postprocess_path(path, lang, force_extension=force_extension)

def _taxonomy_atom_path(self, name, lang, taxonomy):
"""Return path to a classification Atom feed."""
return self._taxonomy_path(name, lang, taxonomy, force_extension='.atom')

def _taxonomy_rss_path(self, name, lang, taxonomy):
"""Return path to a classification RSS feed."""
return self._taxonomy_path(name, lang, taxonomy, force_extension='.xml')

def _register_path_handlers(self, taxonomy):
self.site.register_path_handler('{0}_index'.format(taxonomy.classification_name), lambda name, lang: self._tag_index_path(lang, taxonomy))
self.site.register_path_handler('{0}'.format(taxonomy.classification_name), lambda name, lang: self._tag_path(name, lang, taxonomy))
self.site.register_path_handler('{0}_atom'.format(taxonomy.classification_name), lambda name, lang: self._tag_atom_path(name, lang, taxonomy))
self.site.register_path_handler('{0}_rss'.format(taxonomy.classification_name), lambda name, lang: self._tag_rss_path(name, lang, taxonomy))

def set_site(self, site):
"""Set site, which is a Nikola instance."""
super(TaxonomiesClassifier, self).set_site(site)
# Add hook for after post scanning
blinker.signal("scanned").connect(self._do_classification)
# Register path handlers
for taxonomy in self.plugin_manager.getPluginsOfCategory('Taxonomy'):
self._register_path_handlers(taxonomy)
12 changes: 12 additions & 0 deletions nikola/plugins/task/taxonomies.plugin
@@ -0,0 +1,12 @@
[Core]
name = render_taxonomies
module = taxonomies

[Documentation]
author = Roberto Alsina
version = 1.0
website = https://getnikola.com/
description = Render the taxonomy overviews, classification pages and feeds.

[Nikola]
plugincategory = Task

0 comments on commit 85fac3d

Please sign in to comment.