Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Add tests for metadata extractors
Signed-off-by: Chris Warrick <kwpolska@gmail.com>
  • Loading branch information
Kwpolska committed Jul 5, 2017
1 parent 562533d commit a9fadb2
Show file tree
Hide file tree
Showing 20 changed files with 312 additions and 22 deletions.
33 changes: 31 additions & 2 deletions nikola/metadata_extractors.py
Expand Up @@ -33,7 +33,7 @@
from nikola.utils import unslugify, req_missing

__all__ = ('MetaCondition', 'MetaPriority', 'MetaSource', 'check_conditions')
me_defaults = ('NikolaMetadata', 'YAMLMetadata', 'TOMLMetadata', 'FilenameRegexMetadata')
_default_extractors = []
DEFAULT_EXTRACTOR_NAME = 'nikola'
DEFAULT_EXTRACTOR = None

Expand Down Expand Up @@ -94,6 +94,24 @@ def check_requirements(extractor: MetadataExtractor):
req_missing([pip_name], "use {0} metadata".format(friendly_name), python=True, optional=False)


def classify_extractor(extractor: MetadataExtractor, metadata_extractors_by: dict):
"""Classify an extractor and add it to the metadata_extractors_by dict."""
global DEFAULT_EXTRACTOR
if extractor.name == DEFAULT_EXTRACTOR_NAME:
DEFAULT_EXTRACTOR = extractor
metadata_extractors_by['priority'][extractor.priority].append(extractor)
metadata_extractors_by['source'][extractor.source].append(extractor)
metadata_extractors_by['name'][extractor.name] = extractor
metadata_extractors_by['all'].append(extractor)


def load_defaults(site: 'nikola.nikola.Nikola', metadata_extractors_by: dict):
"""Load default metadata extractors."""
for extractor in _default_extractors:
extractor.site = site
classify_extractor(extractor, metadata_extractors_by)


def is_extractor(extractor) -> bool:
"""Check if a given class is an extractor."""
return isinstance(extractor, MetadataExtractor)
Expand All @@ -104,7 +122,8 @@ def default_metadata_extractors_by() -> dict:
d = {
'priority': {},
'source': {},
'name': {}
'name': {},
'all': []
}

for i in MetaPriority:
Expand All @@ -115,6 +134,13 @@ def default_metadata_extractors_by() -> dict:
return d


def _register_default(extractor: MetadataExtractor) -> MetadataExtractor:
"""Register a default extractor."""
_default_extractors.append(extractor())
return extractor


@_register_default
class NikolaMetadata(MetadataExtractor):
"""Extractor for Nikola-style metadata."""

Expand All @@ -133,6 +159,7 @@ def _extract_metadata_from_text(self, source_text: str) -> dict:
return outdict


@_register_default
class YAMLMetadata(MetadataExtractor):
"""Extractor for YAML metadata."""

Expand All @@ -154,6 +181,7 @@ def _extract_metadata_from_text(self, source_text: str) -> dict:
return meta


@_register_default
class TOMLMetadata(MetadataExtractor):
"""Extractor for TOML metadata."""

Expand All @@ -170,6 +198,7 @@ def _extract_metadata_from_text(self, source_text: str) -> dict:
return toml.loads(source_text[4:])


@_register_default
class FilenameRegexMetadata(MetadataExtractor):
"""Extractor for filename metadata."""

Expand Down
17 changes: 2 additions & 15 deletions nikola/nikola.py
Expand Up @@ -426,7 +426,6 @@ def __init__(self, **config):
self.configured = bool(config)
self.injected_deps = defaultdict(list)
self.shortcode_registry = {}
self.metadata_extractors_all = []
self.metadata_extractors_by = default_metadata_extractors_by()

self.rst_transforms = []
Expand Down Expand Up @@ -830,10 +829,7 @@ def __init__(self, **config):
sys.exit(1)

# Load built-in metadata extractors
for i in metadata_extractors.me_defaults:
inst = getattr(metadata_extractors, i)()
inst.site = self
self._add_metadata_extractor(inst)
metadata_extractors.load_defaults(self, self.metadata_extractors_by)
if metadata_extractors.DEFAULT_EXTRACTOR is None:
utils.LOGGER.error("Could not find default meta extractor ({})".format(
metadata_extractors.DEFAULT_EXTRACTOR_NAME))
Expand Down Expand Up @@ -1022,7 +1018,7 @@ def init_plugins(self, commands_only=False, load_all=False):

# Activate metadata extractors and prepare them for use
for p in self._activate_plugins_of_category("MetadataExtractor"):
self._add_metadata_extractor(p.plugin_object)
metadata_extractors.classify_extractor(p.plugin_object, self.metadata_extractors_by)

self._activate_plugins_of_category("Taxonomy")
self.taxonomy_plugins = {}
Expand Down Expand Up @@ -1203,15 +1199,6 @@ def _activate_plugins_of_category(self, category):
plugins.append(plugin_info)
return plugins

def _add_metadata_extractor(self, extractor: MetadataExtractor):
"""Add a metadata extractor to the site."""
if extractor.name == metadata_extractors.DEFAULT_EXTRACTOR_NAME:
metadata_extractors.DEFAULT_EXTRACTOR = extractor
self.metadata_extractors_all.append(extractor)
self.metadata_extractors_by['priority'][extractor.priority].append(extractor)
self.metadata_extractors_by['source'][extractor.source].append(extractor)
self.metadata_extractors_by['name'][extractor.name] = extractor

def _get_themes(self):
if self._THEMES is None:
try:
Expand Down
7 changes: 7 additions & 0 deletions tests/data/metadata_extractors/f-html-1-compiler.html
@@ -0,0 +1,7 @@
<meta name="title" content="T: HTML, 1, compiler">
<meta name="slug" content="s-html-1-compiler">
<meta name="date" content="2017-07-01 00:00:00 UTC">
<meta name="tags" content="meta,HTML,onefile,compiler">

Content line 1.
Content line 2.
31 changes: 31 additions & 0 deletions tests/data/metadata_extractors/f-ipynb-1-compiler.ipynb
@@ -0,0 +1,31 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Content line 1.\nContent line 2."
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"env": {},
"language": "python",
"name": "python3"
},
"nikola": {
"category": "",
"date": "2017-07-01 00:00:00 UTC",
"description": "",
"link": "",
"slug": "s-ipynb-1-compiler",
"tags": "meta,Jupyter Notebook,onefile,compiler",
"title": "T: Jupyter Notebook, 1, compiler",
"type": "text"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
7 changes: 7 additions & 0 deletions tests/data/metadata_extractors/f-markdown-1-compiler.md
@@ -0,0 +1,7 @@
title: T: Markdown, 1, compiler
slug: s-markdown-1-compiler
date: 2017-07-01 00:00:00 UTC
tags: meta,Markdown,onefile,compiler

Content line 1.
Content line 2.
13 changes: 13 additions & 0 deletions tests/data/metadata_extractors/f-markdown-1-nikola.md
@@ -0,0 +1,13 @@
<!--
.. title: T: Markdown, 1, Nikola
.. slug: s-markdown-1-nikola
.. date: 2017-07-01 00:00:00 UTC
.. tags: meta,Markdown,onefile,Nikola
.. category:
.. link:
.. description:
.. type: text
-->

Content line 1.
Content line 2.
2 changes: 2 additions & 0 deletions tests/data/metadata_extractors/f-markdown-2-nikola.md
@@ -0,0 +1,2 @@
Content line 1.
Content line 2.
7 changes: 7 additions & 0 deletions tests/data/metadata_extractors/f-markdown-2-nikola.meta
@@ -0,0 +1,7 @@
.. title: T: Markdown, 2, Nikola
.. slug: s-markdown-2-nikola
.. date: 2017-07-01 00:00:00 UTC
.. tags: meta,Markdown,twofile,Nikola
.. link:
.. description:
.. type: text
9 changes: 9 additions & 0 deletions tests/data/metadata_extractors/f-rest-1-compiler.rst
@@ -0,0 +1,9 @@
T: reST, 1, compiler
====================

:slug: s-rest-1-compiler
:Date: 2017-07-01 00:00:00 UTC
:tags: meta,reST,onefile,compiler

Content line 1.
Content line 2.
11 changes: 11 additions & 0 deletions tests/data/metadata_extractors/f-rest-1-nikola.rst
@@ -0,0 +1,11 @@
.. title: T: reST, 1, Nikola
.. slug: s-rest-1-nikola
.. date: 2017-07-01 00:00:00 UTC
.. tags: meta,reST,onefile,Nikola
.. category:
.. link:
.. description:
.. type: text
Content line 1.
Content line 2.
8 changes: 8 additions & 0 deletions tests/data/metadata_extractors/f-rest-1-toml.rst
@@ -0,0 +1,8 @@
+++
title = "T: reST, 1, TOML"
slug = "s-rest-1-toml"
date = "2017-07-01 00:00:00 UTC"
tags = "meta,reST,onefile,TOML"
+++
Content line 1.
Content line 2.
8 changes: 8 additions & 0 deletions tests/data/metadata_extractors/f-rest-1-yaml.rst
@@ -0,0 +1,8 @@
---
title: "T: reST, 1, YAML"
slug: s-rest-1-yaml
date: "2017-07-01 00:00:00 UTC"
tags: ["meta", "reST", "onefile", "YAML"]
---
Content line 1.
Content line 2.
7 changes: 7 additions & 0 deletions tests/data/metadata_extractors/f-rest-2-nikola.meta
@@ -0,0 +1,7 @@
.. title: T: reST, 2, Nikola
.. slug: s-rest-2-nikola
.. date: 2017-07-01 00:00:00 UTC
.. tags: meta,reST,twofile,Nikola
.. link:
.. description:
.. type: text
2 changes: 2 additions & 0 deletions tests/data/metadata_extractors/f-rest-2-nikola.rst
@@ -0,0 +1,2 @@
Content line 1.
Content line 2.
6 changes: 6 additions & 0 deletions tests/data/metadata_extractors/f-rest-2-toml.meta
@@ -0,0 +1,6 @@
+++
title = "T: reST, 2, TOML"
slug = "s-rest-2-toml"
date = "2017-07-01 00:00:00 UTC"
tags = "meta,reST,twofile,TOML"
+++
2 changes: 2 additions & 0 deletions tests/data/metadata_extractors/f-rest-2-toml.rst
@@ -0,0 +1,2 @@
Content line 1.
Content line 2.
6 changes: 6 additions & 0 deletions tests/data/metadata_extractors/f-rest-2-yaml.meta
@@ -0,0 +1,6 @@
---
title: "T: reST, 2, YAML"
slug: s-rest-2-yaml
date: "2017-07-01 00:00:00 UTC"
tags: ["meta", "reST", "twofile", "YAML"]
---
2 changes: 2 additions & 0 deletions tests/data/metadata_extractors/f-rest-2-yaml.rst
@@ -0,0 +1,2 @@
Content line 1.
Content line 2.

0 comments on commit a9fadb2

Please sign in to comment.