Use BASE_URL to set "Host:" in /robots.txt

Used by Yandex and Google, possibly others.
getnikola · Sep 30, 2015 · e6b1b5e · e6b1b5e
1 parent 4d2df1c
commit e6b1b5e
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/nikola/plugins/task/robots.py b/nikola/plugins/task/robots.py
@@ -64,10 +64,11 @@ def write_robots():
 
             with io.open(robots_path, 'w+', encoding='utf8') as outf:
                 outf.write("Sitemap: {0}\n\n".format(sitemapindex_url))
+                outf.write("User-Agent: *\n")
                 if kw["robots_exclusions"]:
-                    outf.write("User-Agent: *\n")
                     for loc in kw["robots_exclusions"]:
                         outf.write("Disallow: {0}\n".format(loc))
+                outf.write("Host: {0}\n".format(urlparse(kw["base_url"]).netloc))
 
         yield self.group_task()