Handle robots.tt as suggested in #1757 (untested)

ralsina · ralsina · commit 972d67d44d80 · 2015-05-27T09:51:43.000-03:00
diff --git a/nikola/plugins/command/auto/__init__.py b/nikola/plugins/command/auto/__init__.py
@@ -211,7 +211,10 @@ def serve_static(self, environ, start_response):
         if os.path.isdir(f_path):
             f_path = os.path.join(f_path, self.site.config['INDEX_FILE'])
 
-        if os.path.isfile(f_path):
+        if puri.path == '/robots.txt':
+            start_response(b'200 OK', [(b'Content-type', 'txt/plain')])
+            return '''User-Agent: *\nDisallow: /\n'''
+        elif os.path.isfile(f_path):
             with open(f_path) as fd:
                 start_response(b'200 OK', [(b'Content-type', mimetype)])
                 return self.inject_js(mimetype, fd.read())
