Commit 1920cc68 authored by Tim Peters's avatar Tim Peters

Whitespace normalization.

parent 430236b6
...@@ -49,7 +49,7 @@ default is 1, i.e. aligned. ...@@ -49,7 +49,7 @@ default is 1, i.e. aligned.
""" """
__all__ = ["Chunk"] __all__ = ["Chunk"]
class Chunk: class Chunk:
def __init__(self, file, align = 1, bigendian = 1, inclheader = 0): def __init__(self, file, align = 1, bigendian = 1, inclheader = 0):
import struct import struct
......
...@@ -59,25 +59,25 @@ class RobotFileParser: ...@@ -59,25 +59,25 @@ class RobotFileParser:
elif status>=400: elif status>=400:
self.allow_all = 1 self.allow_all = 1
else: else:
# status < 400 # status < 400
self.parse(connection.getfile().readlines()) self.parse(connection.getfile().readlines())
def parse(self, lines): def parse(self, lines):
"""parse the input lines from a robot.txt file. """parse the input lines from a robot.txt file.
We allow that a user-agent: line is not preceded by We allow that a user-agent: line is not preceded by
one or more blank lines.""" one or more blank lines."""
state = 0 state = 0
linenumber = 0 linenumber = 0
entry = Entry() entry = Entry()
for line in lines: for line in lines:
line = string.strip(line) line = string.strip(line)
linenumber = linenumber + 1 linenumber = linenumber + 1
if not line: if not line:
if state==1: if state==1:
_debug("line %d: warning: you should insert" _debug("line %d: warning: you should insert"
" allow: or disallow: directives below any" " allow: or disallow: directives below any"
" user-agent: line" % linenumber) " user-agent: line" % linenumber)
entry = Entry() entry = Entry()
state = 0 state = 0
elif state==2: elif state==2:
...@@ -98,7 +98,7 @@ class RobotFileParser: ...@@ -98,7 +98,7 @@ class RobotFileParser:
if line[0] == "user-agent": if line[0] == "user-agent":
if state==2: if state==2:
_debug("line %d: warning: you should insert a blank" _debug("line %d: warning: you should insert a blank"
" line before any user-agent" " line before any user-agent"
" directive" % linenumber) " directive" % linenumber)
self.entries.append(entry) self.entries.append(entry)
entry = Entry() entry = Entry()
...@@ -107,14 +107,14 @@ class RobotFileParser: ...@@ -107,14 +107,14 @@ class RobotFileParser:
elif line[0] == "disallow": elif line[0] == "disallow":
if state==0: if state==0:
_debug("line %d: error: you must insert a user-agent:" _debug("line %d: error: you must insert a user-agent:"
" directive before this line" % linenumber) " directive before this line" % linenumber)
else: else:
entry.rulelines.append(RuleLine(line[1], 0)) entry.rulelines.append(RuleLine(line[1], 0))
state = 2 state = 2
elif line[0] == "allow": elif line[0] == "allow":
if state==0: if state==0:
_debug("line %d: error: you must insert a user-agent:" _debug("line %d: error: you must insert a user-agent:"
" directive before this line" % linenumber) " directive before this line" % linenumber)
else: else:
entry.rulelines.append(RuleLine(line[1], 1)) entry.rulelines.append(RuleLine(line[1], 1))
else: else:
...@@ -212,7 +212,7 @@ def _test(): ...@@ -212,7 +212,7 @@ def _test():
print rp.can_fetch('*', 'http://www.musi-cal.com/') print rp.can_fetch('*', 'http://www.musi-cal.com/')
print rp.can_fetch('Musi-Cal-Robot/1.0', print rp.can_fetch('Musi-Cal-Robot/1.0',
'http://www.musi-cal.com/cgi-bin/event-search' 'http://www.musi-cal.com/cgi-bin/event-search'
'?city=San+Francisco') '?city=San+Francisco')
if __name__ == '__main__': if __name__ == '__main__':
_test() _test()
from test_support import verify, verbose, TestFailed from test_support import verify, verbose, TestFailed
import sys import sys
...@@ -8,7 +7,7 @@ def check_all(_modname): ...@@ -8,7 +7,7 @@ def check_all(_modname):
"%s has no __all__ attribute" % _modname) "%s has no __all__ attribute" % _modname)
exec "del %s" % _modname exec "del %s" % _modname
exec "from %s import *" % _modname exec "from %s import *" % _modname
_keys = locals().keys() _keys = locals().keys()
_keys.remove("_modname") _keys.remove("_modname")
_keys.sort() _keys.sort()
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment