merged master

getnikola · Jan 29, 2016 · 82fb86d · 82fb86d
2 parents 304b9e5 + a25b5ea
commit 82fb86d
Show file tree

Hide file tree

Showing 5 changed files with 144 additions and 12 deletions.
diff --git a/CHANGES.txt b/CHANGES.txt
@@ -10,7 +10,8 @@ Features
   if ``GITHUB_COMMIT_SOURCE`` is set to True (Issue #2186)
 * Hugo-like shortcodes (Issue #1707)
 * New Galician translation
-* New PRESERVE_EXIF_DATA to copy EXIF when resizing images (Issue #2204)
+* New PRESERVE_EXIF_DATA option to copy EXIF when resizing images (Issue #2204)
+* New facilities for data persistence and data caching (Issues #2209 and #2009)
 
 Bugfixes
 --------

diff --git a/docs/extending.txt b/docs/extending.txt
@@ -592,3 +592,29 @@ So, for example::
 Will cause a call like this::
 
      foo_handler("bar", "beep", baz="bat", data="Some text", site=whatever)
+
+State and Cache
+===============
+
+Sometimes your plugins will need to cache things to speed up further actions. Here are the conventions for that:
+
+* If it's a file, put it somewhere in ```self.site.config['CACHE_FOLDER']``` (defaults to ```cache/```.
+* If it's a value, use ```self.site.cache.set(key, value)``` to set it and ```self.site.cache.get(key)``` to get it.
+  The key should be a string, the value should be json-encodable (so, be careful with datetime objects)
+
+The values and files you store there can **and will** be deleted sometimes by the user. They should always be
+things you can reconstruct without lossage. They are throwaways.
+
+On the other hand, sometimes you want to save something that is **not** a throwaway. These are things that may
+change the output, so the user should not delete them. We call that **state**. To save state:
+
+* If it's a file, put it somewhere in the working directory. Try not to do that please.
+* If it's a value, use ```self.site.state.set(key, value)``` to set it and ```self.state.cache.get(key)``` to get it.
+  The key should be a string, the value should be json-encodable (so, be careful with datetime objects)
+
+The ```cache``` and ```state``` objects are rather simplistic, and that's intentional. They have no default values: if
+the key is not there, you will get ```None``` and like it. They are meant to be both threadsafe, but hey, who can
+guarantee that sort of thing?
+
+There are no sections, and no access protection, so let's not use it to store passwords and such. Use responsibly.
+
diff --git a/nikola/nikola.py b/nikola/nikola.py
@@ -57,6 +57,7 @@
 from blinker import signal
 
 from .post import Post  # NOQA
+from .state import Persistor
 from . import DEBUG, utils, shortcodes
 from .plugin_categories import (
     Command,
@@ -814,6 +815,12 @@ def __init__(self, **config):
 
         self._set_global_context()
 
+        # Set persistent state facility
+        self.state = Persistor(os.path.join('state_data.json'))
+
+        # Set cache facility
+        self.cache = Persistor(os.path.join(self.config['CACHE_FOLDER'], 'cache_data.json'))
+
     def init_plugins(self, commands_only=False):
         """Load plugins as needed."""
         self.plugin_manager = PluginManager(categories_filter={

diff --git a/nikola/plugins/command/deploy.py b/nikola/plugins/command/deploy.py
@@ -30,14 +30,15 @@
 import io
 from datetime import datetime
 from dateutil.tz import gettz
+import dateutil
 import os
 import subprocess
 import time
 
 from blinker import signal
 
 from nikola.plugin_categories import Command
-from nikola.utils import get_logger, remove_file, unicode_str, makedirs, STDERR_HANDLER
+from nikola.utils import get_logger, remove_file, makedirs, STDERR_HANDLER
 
 
 class CommandDeploy(Command):
@@ -55,6 +56,29 @@ def _execute(self, command, args):
         self.logger = get_logger('deploy', STDERR_HANDLER)
         # Get last successful deploy date
         timestamp_path = os.path.join(self.site.config['CACHE_FOLDER'], 'lastdeploy')
+
+        # Get last-deploy from persistent state
+        last_deploy = self.site.state.get('last_deploy')
+        if last_deploy is None:
+            # If there is a last-deploy saved, move it to the new state persistence thing
+            # FIXME: remove in Nikola 8
+            if os.path.isfile(timestamp_path):
+                try:
+                    with io.open(timestamp_path, 'r', encoding='utf8') as inf:
+                        last_deploy = dateutil.parser.parse(inf.read())
+                        clean = False
+                except (IOError, Exception) as e:
+                    self.logger.debug("Problem when reading `{0}`: {1}".format(timestamp_path, e))
+                    last_deploy = datetime(1970, 1, 1)
+                    clean = True
+                os.unlink(timestamp_path)  # Remove because from now on it's in state
+            else:  # Just a default
+                last_deploy = datetime(1970, 1, 1)
+                clean = True
+        else:
+            last_deploy = dateutil.parser.parse(last_deploy)
+            clean = False
+
         if self.site.config['COMMENT_SYSTEM_ID'] == 'nikolademo':
             self.logger.warn("\nWARNING WARNING WARNING WARNING\n"
                              "You are deploying using the nikolademo Disqus account.\n"
@@ -102,22 +126,13 @@ def _execute(self, command, args):
                     return e.returncode
 
         self.logger.info("Successful deployment")
-        try:
-            with io.open(timestamp_path, 'r', encoding='utf8') as inf:
-                last_deploy = datetime.strptime(inf.read().strip(), "%Y-%m-%dT%H:%M:%S.%f")
-                clean = False
-        except (IOError, Exception) as e:
-            self.logger.debug("Problem when reading `{0}`: {1}".format(timestamp_path, e))
-            last_deploy = datetime(1970, 1, 1)
-            clean = True
 
         new_deploy = datetime.utcnow()
         self._emit_deploy_event(last_deploy, new_deploy, clean, undeployed_posts)
 
         makedirs(self.site.config['CACHE_FOLDER'])
         # Store timestamp of successful deployment
-        with io.open(timestamp_path, 'w+', encoding='utf8') as outf:
-            outf.write(unicode_str(new_deploy.isoformat()))
+        self.site.state.set('last_deploy', new_deploy.isoformat())
 
     def _emit_deploy_event(self, last_deploy, new_deploy, clean=False, undeployed=None):
         """Emit events for all timeline entries newer than last deploy.

diff --git a/nikola/state.py b/nikola/state.py
@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012-2016 Roberto Alsina and others.
+
+# Permission is hereby granted, free of charge, to any
+# person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the
+# Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the
+# Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice
+# shall be included in all copies or substantial portions of
+# the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
+# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+"""Persistent state implementation."""
+
+import json
+import os
+import shutil
+import tempfile
+import threading
+
+from . import utils
+
+
+class Persistor():
+    """Persist stuff in a place.
+
+    This is an intentionally dumb implementation. It is *not* meant to be
+    fast, or useful for arbitrarily large data. Use lightly.
+
+    Intentionally it has no namespaces, sections, etc. Use as a
+    responsible adult.
+    """
+
+    def __init__(self, path):
+        """Where do you want it persisted."""
+        self._path = path
+        utils.makedirs(os.path.dirname(path))
+        self._local = threading.local()
+        self._local.data = {}
+
+    def get(self, key):
+        """Get data stored in key."""
+        self._read()
+        return self._local.data.get(key)
+
+    def set(self, key, value):
+        """Store value in key."""
+        self._read()
+        self._local.data[key] = value
+        self._save()
+
+    def delete(self, key):
+        """Delete key and the value it contains."""
+        self._read()
+        if key in self._local.data:
+            self._local.data.pop(key)
+        self._save()
+
+    def _read(self):
+        if os.path.isfile(self._path):
+            with open(self._path) as inf:
+                self._local.data = json.load(inf)
+
+    def _save(self):
+        dname = os.path.dirname(self._path)
+        with tempfile.NamedTemporaryFile(dir=dname, delete=False) as outf:
+            tname = outf.name
+            json.dump(self._local.data, outf, sort_keys=True, indent=2)
+        shutil.move(tname, self._path)