rules = []
    for rule in exclusions:
        rules.append('Disallow: {0}'.format(rule))

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import urllib.robotparser

def main():
    r = urllib.robotparser.RobotFileParser()
    r.parse(["User-Agent: *", "Disallow: /a.html", "Disallow: /b.html"])
    print(r.can_fetch("*", '/a.html')) # False
    print(r.can_fetch("*", '/b.html')) # False
    print(r.can_fetch("*", '/c.html')) # True

    r = urllib.robotparser.RobotFileParser()
    r.parse(["User-Agent: *", "Disallow: /a.html"])
    print(r.can_fetch("*", '/a.html')) # False
    print(r.can_fetch("*", '/b.html')) # True
    print(r.can_fetch("*", '/c.html')) # True
    r.parse(["User-Agent: *", "Disallow: /b.html"]) # Ignored
    print(r.can_fetch("*", '/a.html')) # False
    print(r.can_fetch("*", '/b.html')) # True
    print(r.can_fetch("*", '/c.html')) # True

if __name__ == '__main__':

    args = main()