Split tasks into stages, with three (four) default stages: EarlyTask,…

… Task and LateTask. Scanning posts is done in additional stage between EarlyTask and Task.
getnikola · Jan 18, 2015 · 6b5d531 · 6b5d531
1 parent 2b34513
commit 6b5d531
Show file tree

Hide file tree

Showing 12 changed files with 119 additions and 24 deletions.
diff --git a/nikola/__main__.py b/nikola/__main__.py
@@ -237,6 +237,24 @@ def __init__(self, nikola, quiet=False):
         self.nikola = nikola
         self.quiet = quiet
 
+    def _get_stage_info(self, level):
+        if level == -1:
+            name = 'pre_scanning'
+            doc = 'Group of tasks to be run before scanning for posts'
+        elif level == 0:
+            name = 'scanning'
+            doc = 'Task scanning for posts'
+        elif level == 10:
+            name = 'render_site'
+            doc = 'Group of tasks to render the site.'
+        elif level == 100:
+            name = 'post_render'
+            doc = 'Group of tasks to be executed after site is rendered.'
+        else:
+            name = 'level_{0}_wait'.format(level)
+            doc = 'Level {0} tasks'.format(level)
+        return name, doc
+
     def load_tasks(self, cmd, opt_values, pos_args):
         if self.quiet:
             DOIT_CONFIG = {
@@ -248,15 +266,16 @@ def load_tasks(self, cmd, opt_values, pos_args):
                 'reporter': ExecutedOnlyReporter,
                 'outfile': sys.stderr,
             }
-        DOIT_CONFIG['default_tasks'] = ['render_site', 'post_render']
-        tasks = generate_tasks(
-            'render_site',
-            self.nikola.gen_tasks('render_site', "Task", 'Group of tasks to render the site.'))
-        latetasks = generate_tasks(
-            'post_render',
-            self.nikola.gen_tasks('post_render', "LateTask", 'Group of tasks to be executed after site is rendered.'))
+        DOIT_CONFIG['default_tasks'] = []
+        tasks = []
+        for level in self.nikola.get_task_stages():
+            name, doc = self._get_stage_info(level)
+            DOIT_CONFIG['default_tasks'].append(name)
+            tasks.extend(generate_tasks(
+                name,
+                self.nikola.gen_tasks(name, level, doc)))
         signal('initialized').send(self.nikola)
-        return tasks + latetasks, DOIT_CONFIG
+        return tasks, DOIT_CONFIG
 
 
 class DoitNikola(DoitMain):

diff --git a/nikola/nikola.py b/nikola/nikola.py
@@ -71,7 +71,9 @@
 from .post import Post
 from . import utils
 from .plugin_categories import (
+    BaseTask,
     Command,
+    EarlyTask,
     LateTask,
     PageCompiler,
     RestExtension,
@@ -614,8 +616,7 @@ def __init__(self, **config):
 
         self.plugin_manager = PluginManager(categories_filter={
             "Command": Command,
-            "Task": Task,
-            "LateTask": LateTask,
+            "BaseTask": BaseTask,
             "TemplateSystem": TemplateSystem,
             "PageCompiler": PageCompiler,
             "TaskMultiplier": TaskMultiplier,
@@ -654,8 +655,14 @@ def __init__(self, **config):
             plugin_info.plugin_object.short_help = plugin_info.description
             self._commands[plugin_info.name] = plugin_info.plugin_object
 
-        self._activate_plugins_of_category("Task")
-        self._activate_plugins_of_category("LateTask")
+        task_plugins = self._activate_plugins_of_category("BaseTask")
+        self.task_stages = defaultdict(list)
+        for plugin_info in task_plugins:
+            if type(plugin_info.plugin_object) == EarlyTask: continue
+            if type(plugin_info.plugin_object) == Task: continue
+            if type(plugin_info.plugin_object) == LateTask: continue
+            stage = plugin_info.plugin_object.get_stage()
+            self.task_stages[stage].append(plugin_info.plugin_object)
         self._activate_plugins_of_category("TaskMultiplier")
 
         compilers = defaultdict(set)
@@ -1239,7 +1246,10 @@ def clean_task_paths(self, task):
             task['targets'] = [os.path.normpath(t) for t in targets]
         return task
 
-    def gen_tasks(self, name, plugin_category, doc=''):
+    def get_task_stages(self):
+        return sorted(list(self.task_stages.keys()))
+
+    def gen_tasks(self, name, stage, doc=''):
 
         def flatten(task):
             if isinstance(task, dict):
@@ -1250,8 +1260,8 @@ def flatten(task):
                         yield ft
 
         task_dep = []
-        for pluginInfo in self.plugin_manager.getPluginsOfCategory(plugin_category):
-            for task in flatten(pluginInfo.plugin_object.gen_tasks()):
+        for plugin_object in self.task_stages[stage]:
+            for task in flatten(plugin_object.gen_tasks()):
                 assert 'basename' in task
                 task = self.clean_task_paths(task)
                 yield task
@@ -1262,8 +1272,8 @@ def flatten(task):
                         yield self.clean_task_paths(task)
                     if flag:
                         task_dep.append('{0}_{1}'.format(name, multi.plugin_object.name))
-            if pluginInfo.plugin_object.is_default:
-                task_dep.append(pluginInfo.plugin_object.name)
+            if plugin_object.is_default:
+                task_dep.append(plugin_object.name)
         yield {
             'basename': name,
             'doc': doc,

diff --git a/nikola/plugin_categories.py b/nikola/plugin_categories.py
@@ -141,6 +141,10 @@ class BaseTask(BasePlugin):
     # the others have to be specifie in the command line.
     is_default = True
 
+    def get_stage(self):
+        """At which level to run."""
+        raise NotImplementedError()
+
     def gen_tasks(self):
         """Task generator."""
         raise NotImplementedError()
@@ -154,17 +158,32 @@ def group_task(self):
         }
 
 
+class EarlyTask(BaseTask):
+    """Plugins of this type are task generators which are executed before tasks are generated."""
+
+    name = "dummy_task"
+
+    def get_stage(self):
+        return -1
+
+
 class Task(BaseTask):
     """Plugins of this type are task generators."""
 
     name = "dummy_task"
 
+    def get_stage(self):
+        return 10
+
 
 class LateTask(BaseTask):
     """Plugins of this type are executed after all plugins of type Task."""
 
     name = "dummy_latetask"
 
+    def get_stage(self):
+        return 100
+
 
 class TemplateSystem(BasePlugin):
     """Plugins of this type wrap templating systems."""

diff --git a/nikola/plugins/task/archive.py b/nikola/plugins/task/archive.py
@@ -114,7 +114,6 @@ def gen_tasks(self):
             "strip_indexes": self.site.config['STRIP_INDEXES'],
             "index_file": self.site.config['INDEX_FILE'],
         }
-        self.site.scan_posts()
         yield self.group_task()
         # TODO add next/prev links for years
         if (kw['create_monthly_archive'] and kw['create_single_archive']) and not kw['create_full_archives']:

diff --git a/nikola/plugins/task/indexes.py b/nikola/plugins/task/indexes.py
@@ -42,7 +42,6 @@ def set_site(self, site):
         return super(Indexes, self).set_site(site)
 
     def gen_tasks(self):
-        self.site.scan_posts()
         yield self.group_task()
 
         kw = {

diff --git a/nikola/plugins/task/pages.py b/nikola/plugins/task/pages.py
@@ -43,7 +43,6 @@ def gen_tasks(self):
             "show_untranslated_posts": self.site.config['SHOW_UNTRANSLATED_POSTS'],
             "demote_headers": self.site.config['DEMOTE_HEADERS'],
         }
-        self.site.scan_posts()
         yield self.group_task()
         for lang in kw["translations"]:
             for post in self.site.timeline:

diff --git a/nikola/plugins/task/posts.py b/nikola/plugins/task/posts.py
@@ -47,7 +47,6 @@ class RenderPosts(Task):
 
     def gen_tasks(self):
         """Build HTML fragments from metadata and text."""
-        self.site.scan_posts()
         kw = {
             "translations": self.site.config["TRANSLATIONS"],
             "timeline": self.site.timeline,

diff --git a/nikola/plugins/task/rss.py b/nikola/plugins/task/rss.py
@@ -62,7 +62,6 @@ def gen_tasks(self):
             "rss_read_more_link": self.site.config["RSS_READ_MORE_LINK"],
             "rss_links_append_query": self.site.config["RSS_LINKS_APPEND_QUERY"],
         }
-        self.site.scan_posts()
         # Check for any changes in the state of use_in_feeds for any post.
         # Issue #934
         kw['use_in_feeds_status'] = ''.join(

diff --git a/nikola/plugins/task/scan_posts.plugin b/nikola/plugins/task/scan_posts.plugin
@@ -0,0 +1,10 @@
+[Core]
+Name = scan_posts
+Module = scan_posts
+
+[Documentation]
+Author = Roberto Alsina and others
+Version = 1.0
+Website = http://getnikola.com
+Description = Scans posts.
+
diff --git a/nikola/plugins/task/scan_posts.py b/nikola/plugins/task/scan_posts.py
@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012-2015 Roberto Alsina and others.
+
+# Permission is hereby granted, free of charge, to any
+# person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the
+# Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the
+# Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice
+# shall be included in all copies or substantial portions of
+# the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
+# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+from copy import copy
+
+from nikola.plugin_categories import BaseTask
+
+
+class ScanPosts(BaseTask):
+    """Just load tasks."""
+
+    name = "scan_posts"
+
+    def get_stage(self):
+        return 0
+
+    def gen_tasks(self):
+        """Scan for posts."""
+        self.site.scan_posts(True)
+
+        yield self.group_task()
diff --git a/nikola/plugins/task/sources.py b/nikola/plugins/task/sources.py
@@ -52,7 +52,6 @@ def gen_tasks(self):
             "show_untranslated_posts": self.site.config['SHOW_UNTRANSLATED_POSTS'],
         }
 
-        self.site.scan_posts()
         yield self.group_task()
         if self.site.config['COPY_SOURCES']:
             for lang in kw["translations"]:

diff --git a/nikola/plugins/task/tags.py b/nikola/plugins/task/tags.py
@@ -81,7 +81,6 @@ def gen_tasks(self):
             "index_file": self.site.config['INDEX_FILE'],
         }
 
-        self.site.scan_posts()
         yield self.group_task()
 
         yield self.list_tags_page(kw)