Restructured Checker class to get rid of 'ext' table.

Links are now either in 'todo' or 'done', and ext links
are hadled more like local links except that no further
links are gathered (and sometimes they aren't checked,
e.g. for mailto and news URLs).  The -x option reverses
its meaning: it disables checking of ext links (they are
moved to 'done' without checking).  A new 'errors' table
collects pages with bad links as we go -- redundant,
but useful for the GUI version which needs to report
this as we go.  Some new methods, including reset().
New checkpoint format.

Adapted the GUI to the changes in the Checker class.
Added Quit and "Start over" buttons, and a checkbox
to disable checking external links.  The details
window now also shows bad links emanating from the
selected page.  Miscellaneous small chages.

Tools/webchecker/wcgui.py

@@ -6,42 +6,37 @@ This works as a Grail applet too!  E.g.
 
   <APPLET CODE=wcgui.py NAME=CheckerWindow></APPLET>
 
-Checkpoints are not (yet?) supported.
+Checkpoints are not (yet???  ever???) supported.
 
 User interface:
 
 Enter a root to check in the text entry box.  To enter more than one root, 
-enter each root and press <Return>.
+enter them one at a time and press <Return> for each one.
 
 Command buttons Start, Stop and "Check one" govern the checking process in 
 the obvious way.  Start and "Check one" also enter the root from the text 
-entry box if one is present.
+entry box if one is present.  There's also a check box (enabled by default)
+to decide whether actually to follow external links (since this can slow
+the checking down considerably).  Finally there's a Quit button.
 
 A series of checkbuttons determines whether the corresponding output panel 
 is shown.  List panels are also automatically shown or hidden when their 
 status changes between empty to non-empty.  There are six panels:
 
 Log        -- raw output from the checker (-v, -q affect this)
-To check   -- local links discovered but not yet checked
-Off site   -- links discovered that point off site
-Checked    -- local links that have been checked
+To check   -- links discovered but not yet checked
+Checked    -- links that have been checked
 Bad links  -- links that failed upon checking
+Errors     -- pages containing at least one bad link
 Details    -- details about one URL; double click on a URL in any of
-              the aboce list panels (not in Log) will show that URL
-
-XXX There ought to be a list of pages known to contain at least one
-bad link.
-
-XXX The checking of off site links should be made more similar to the 
-checking of local links (even if they are checked after all local links are 
-checked).
+              the above list panels (not in Log) will show details
+              for that URL
 
 Use your window manager's Close command to quit.
 
 Command line options:
 
 -m bytes  -- skip HTML pages larger than this size (default %(MAXPAGE)d)
--n        -- reports only, no checking (use with -R)
 -q        -- quiet operation (also suppresses external links report)
 -v        -- verbose operation; repeating -v will increase verbosity
 
@@ -50,11 +45,11 @@ Command line arguments:
 rooturl   -- URL to start checking
              (default %(DEFROOT)s)
 
-XXX The command line options should all be GUI accessible.
+XXX The command line options (-m, -q, -v) should be GUI accessible.
 
 XXX The roots should be visible as a list (?).
 
-XXX The multipanel user interface is bogus.
+XXX The multipanel user interface is clumsy.
 
 """
 
@@ -104,39 +99,66 @@ class CheckerWindow(webchecker.Checker):
 
     def __init__(self, parent, root=webchecker.DEFROOT):
 	self.__parent = parent
-	self.__controls = Frame(parent)
-	self.__controls.pack(side=TOP, fill=X)
-	self.__label = Label(self.__controls, text="Root URL:")
+
+	self.__topcontrols = Frame(parent)
+	self.__topcontrols.pack(side=TOP, fill=X)
+	self.__label = Label(self.__topcontrols, text="Root URL:")
 	self.__label.pack(side=LEFT)
-	self.__rootentry = Entry(self.__controls, width=60)
+	self.__rootentry = Entry(self.__topcontrols, width=60)
 	self.__rootentry.pack(side=LEFT)
 	self.__rootentry.bind('<Return>', self.enterroot)
 	self.__rootentry.focus_set()
+
+	self.__controls = Frame(parent)
+	self.__controls.pack(side=TOP, fill=X)
 	self.__running = 0
 	self.__start = Button(self.__controls, text="Run", command=self.start)
 	self.__start.pack(side=LEFT)
 	self.__stop = Button(self.__controls, text="Stop", command=self.stop,
 			     state=DISABLED)
 	self.__stop.pack(side=LEFT)
-	self.__step = Button(self.__controls, text="Check one", command=self.step)
+	self.__step = Button(self.__controls, text="Check one",
+			     command=self.step)
 	self.__step.pack(side=LEFT)
+	self.__cv = BooleanVar()
+	self.__cv.set(1)
+	self.__checkext = Checkbutton(self.__controls, variable=self.__cv,
+				      text="Check nonlocal links")
+	self.__checkext.pack(side=LEFT)
+	self.__reset = Button(self.__controls, text="Start over", command=self.reset)
+	self.__reset.pack(side=LEFT)
+	if __name__ == '__main__': # No Quit button under Grail!
+	    self.__quit = Button(self.__controls, text="Quit",
+				 command=self.__parent.quit)
+	    self.__quit.pack(side=RIGHT)
+
 	self.__status = Label(parent, text="Status: initial", anchor=W)
 	self.__status.pack(side=TOP, fill=X)
-	self.__checking = Label(parent, text="Checking: none", anchor=W)
+	self.__checking = Label(parent, text="Idle", anchor=W)
 	self.__checking.pack(side=TOP, fill=X)
 	self.__mp = mp = MultiPanel(parent)
 	sys.stdout = self.__log = LogPanel(mp, "Log")
 	self.__todo = ListPanel(mp, "To check", self.showinfo)
-	self.__ext = ListPanel(mp, "Off site", self.showinfo)
 	self.__done = ListPanel(mp, "Checked", self.showinfo)
 	self.__bad = ListPanel(mp, "Bad links", self.showinfo)
+	self.__errors = ListPanel(mp, "Pages w/ bad links", self.showinfo)
 	self.__details = LogPanel(mp, "Details")
-	self.__extodo = []
 	webchecker.Checker.__init__(self)
+	del self.checkext # See __getattr__ below
 	if root:
 	    root = string.strip(str(root))
 	    if root:
 		self.suggestroot(root)
+	self.newstatus()
+
+    def __getattr__(self, name):
+	if name != 'checkext': raise AttributeError, name
+	return self.__cv.get()
+
+    def reset(self):
+	webchecker.Checker.reset(self)
+	for p in self.__todo, self.__done, self.__bad, self.__errors:
+	    p.clear()
 
     def suggestroot(self, root):
 	self.__rootentry.delete(0, END)
@@ -147,7 +169,10 @@ class CheckerWindow(webchecker.Checker):
 	root = self.__rootentry.get()
 	root = string.strip(root)
 	if root:
+	    self.__checking.config(text="Adding root "+root)
+	    self.__checking.update_idletasks()
 	    self.addroot(root)
+	    self.__checking.config(text="Idle")
 	    try:
 		i = self.__todo.items.index(root)
 	    except (ValueError, IndexError):
@@ -167,7 +192,7 @@ class CheckerWindow(webchecker.Checker):
 	self.go()
 
     def stop(self):
-	self.__stop.config(state=DISABLED)
+	self.__stop.config(state=DISABLED, relief=SUNKEN)
 	self.__running = 0
 
     def step(self):
@@ -181,8 +206,9 @@ class CheckerWindow(webchecker.Checker):
 	if self.__running:
 	    self.__parent.after_idle(self.dosomething)
 	else:
-	    self.__checking.config(text="Checking: none")
+	    self.__checking.config(text="Idle")
 	    self.__start.config(state=NORMAL, relief=RAISED)
+	    self.__stop.config(state=DISABLED, relief=RAISED)
 	    self.__step.config(state=NORMAL, relief=RAISED)
 
     __busy = 0
@@ -199,18 +225,9 @@ class CheckerWindow(webchecker.Checker):
 		self.__todo.list.select_set(i)
 	    self.__todo.list.yview(i)
 	    url = self.__todo.items[i]
-	    self.__checking.config(text="Checking: "+url)
+	    self.__checking.config(text="Checking "+url)
 	    self.__parent.update()
 	    self.dopage(url)
-	elif self.__extodo:
-	    # XXX Should have an indication of these in the todo window...
-	    ##i = random.randint(0, len(self.__extodo)-1)
-	    i = 0
-	    url = self.__extodo[i]
-	    del self.__extodo[i]
-	    self.__checking.config(text="Checking: "+url)
-	    self.__parent.update()
-	    self.checkextpage(url)
 	else:
 	    self.stop()
 	self.__busy = 0
@@ -219,33 +236,34 @@ class CheckerWindow(webchecker.Checker):
     def showinfo(self, url):
 	d = self.__details
 	d.clear()
-	d.write("URL:    %s\n" % url)
+	d.put("URL:    %s\n" % url)
 	if self.bad.has_key(url):
-	    d.write("Error:  %s\n" % str(self.bad[url]))
+	    d.put("Error:  %s\n" % str(self.bad[url]))
 	if url in self.roots:
-	    d.write("Note:   This is a root URL\n")
+	    d.put("Note:   This is a root URL\n")
 	if self.done.has_key(url):
-	    d.write("Status: checked\n")
+	    d.put("Status: checked\n")
 	    o = self.done[url]
 	elif self.todo.has_key(url):
-	    d.write("Status: to check\n")
+	    d.put("Status: to check\n")
 	    o = self.todo[url]
-	elif self.ext.has_key(url):
-	    d.write("Status: off site\n")
-	    o = self.ext[url]
 	else:
-	    d.write("Status: unknown (!)\n")
+	    d.put("Status: unknown (!)\n")
 	    o = []
+	if self.errors.has_key(url):
+	    d.put("Bad links from this page:\n")
+	    for triple in self.errors[url]:
+		link, rawlink, msg = triple
+		d.put("  HREF  %s" % link)
+		if link != rawlink: d.put(" (%s)" %rawlink)
+		d.put("\n")
+		d.put("  error %s\n" % str(msg))
 	self.__mp.showpanel("Details")
 	for source, rawlink in o:
-	    d.write("Origin: %s" % source)
+	    d.put("Origin: %s" % source)
 	    if rawlink != url:
-		d.write(" (%s)" % rawlink)
-	    d.write("\n")
-
-    def newstatus(self):
-	self.__status.config(text="Status: "+self.status()[1:-1])
-	self.__parent.update()
+		d.put(" (%s)" % rawlink)
+	    d.put("\n")
 
     def setbad(self, url, msg):
 	webchecker.Checker.setbad(self, url, msg)
@@ -257,14 +275,8 @@ class CheckerWindow(webchecker.Checker):
 	self.__bad.remove(url)
 	self.newstatus()
 
-    def newextlink(self, url, origin):
-	webchecker.Checker.newextlink(self, url, origin)
-	self.__extodo.append(url)
-	self.__ext.insert(url)
-	self.newstatus()
-
-    def newintlink(self, url, origin):
-	webchecker.Checker.newintlink(self, url, origin)
+    def newlink(self, url, origin):
+	webchecker.Checker.newlink(self, url, origin)
 	if self.done.has_key(url):
 	    self.__done.insert(url)
 	elif self.todo.has_key(url):
@@ -277,6 +289,15 @@ class CheckerWindow(webchecker.Checker):
 	self.__todo.remove(url)
 	self.newstatus()
 
+    def seterror(self, url, triple):
+	webchecker.Checker.seterror(self, url, triple)
+	self.__errors.insert(url)
+	self.newstatus()
+
+    def newstatus(self):
+	self.__status.config(text="Status: "+self.status())
+	self.__parent.update()
+
 
 class ListPanel:
 
@@ -292,6 +313,11 @@ class ListPanel:
 	    self.list.bind('<Double-Button-1>', self.doubleclick)
 	self.items = []
 
+    def clear(self):
+	self.items = []
+	self.list.delete(0, END)
+	self.mp.hidepanel(self.name)
+
     def doubleclick(self, event):
 	l = self.selectedindices()
 	if l:
@@ -309,7 +335,7 @@ class ListPanel:
 	    # (I tried sorting alphabetically, but the display is too jumpy)
 	    i = len(self.items)
 	    self.list.insert(i, url)
-##	    self.list.yview(i)
+	    self.list.yview(i)
 	    self.items.insert(i, url)
 
     def remove(self, url):
@@ -342,6 +368,11 @@ class LogPanel:
 	self.text.delete("1.0", END)
 	self.text.yview("1.0")
 
+    def put(self, s):
+	self.text.insert(END, s)
+	if '\n' in s:
+	    self.text.yview(END)
+
     def write(self, s):
 	self.text.insert(END, s)
 	if '\n' in s:

Tools/webchecker/webchecker.py

@@ -19,12 +19,10 @@ most of it checked.  In fact the default works this way if your local
 web tree is located at /usr/local/etc/httpd/htdpcs (the default for
 the NCSA HTTP daemon and probably others).
 
-Reports printed:
+Report printed:
 
-When done, it reports links to pages outside the web (unless -q is
-specified), and pages with bad links within the subweb.  When
-interrupted, it print those same reports for the pages that it has
-checked already.
+When done, it reports pages with bad links within the subweb.  When
+interrupted, it reports for the pages that it has checked already.
 
 In verbose mode, additional messages are printed during the
 information gathering phase.  By default, it prints a summary of its
@@ -50,31 +48,29 @@ overwritten, but all work done in the current run is lost.
 
 Miscellaneous:
 
+- You may find the (Tk-based) GUI version easier to use.  See wcgui.py.
+
 - Webchecker honors the "robots.txt" convention.  Thanks to Skip
 Montanaro for his robotparser.py module (included in this directory)!
 The agent name is hardwired to "webchecker".  URLs that are disallowed
 by the robots.txt file are reported as external URLs.
 
-- Because the HTML parser is a bit slow, very large HTML files are
+- Because the SGML parser is a bit slow, very large SGML files are
 skipped.  The size limit can be set with the -m option.
 
-- Before fetching a page, it guesses its type based on its extension.
-If it is a known extension and the type is not text/html, the page is
-not fetched.  This is a huge optimization but occasionally it means
-links can be missed, and such links aren't checked for validity
-(XXX!).  The mimetypes.py module (also in this directory) has a
-built-in table mapping most currently known suffixes, and in addition
-attempts to read the mime.types configuration files in the default
-locations of Netscape and the NCSA HTTP daemon.
+- When the server or protocol does not tell us a file's type, we guess
+it based on the URL's suffix.  The mimetypes.py module (also in this
+directory) has a built-in table mapping most currently known suffixes,
+and in addition attempts to read the mime.types configuration files in
+the default locations of Netscape and the NCSA HTTP daemon.
 
-- It only follows links indicated by <A> tags.  It doesn't follow
-links in <FORM> or <IMG> or whatever other tags might contain
-hyperlinks.  It does honor the <BASE> tag.
+- We follows links indicated by <A>, <FRAME> and <IMG> tags.  We also
+honor the <BASE> tag.
 
-- Checking external links is not done by default; use -x to enable
-this feature.  This is done because checking external links usually
-takes a lot of time.  When enabled, this check is executed during the
-report generation phase (even when the report is silent).
+- Checking external links is now done by default; use -x to *disable*
+this feature.  External links are now checked during normal
+processing.  (XXX The status of a checked link could be categorized
+better.  Later...)
 
 
 Usage: webchecker.py [option] ... [rooturl] ...
@@ -88,7 +84,7 @@ Options:
 -q        -- quiet operation (also suppresses external links report)
 -r number -- number of links processed per round (default %(ROUNDSIZE)d)
 -v        -- verbose operation; repeating -v will increase verbosity
--x        -- check external links (during report phase)
+-x        -- don't check external links (these are often slow to check)
 
 Arguments:
 
@@ -100,7 +96,7 @@ rooturl   -- URL to start checking
 # ' Emacs bait
 
 
-__version__ = "0.3"
+__version__ = "0.4"
 
 
 import sys
@@ -137,7 +133,7 @@ def main():
     global verbose, maxpage, roundsize
     dumpfile = DUMPFILE
     restart = 0
-    checkext = 0
+    checkext = 1
     norun = 0
 
     try:
@@ -163,7 +159,7 @@ def main():
 	if o == '-v':
 	    verbose = verbose + 1
 	if o == '-x':
-	    checkext = 1
+	    checkext = not checkext
 
     if verbose > 0:
 	print AGENTNAME, "version", __version__
@@ -178,7 +174,7 @@ def main():
 	    print "Done."
 	    print "Root:", string.join(c.roots, "\n      ")
     else:
-	c = Checker()
+	c = Checker(checkext)
 	if not args:
 	    args.append(DEFROOT)
 
@@ -193,7 +189,7 @@ def main():
 		print "[run interrupted]"
 
     try:
-	c.report(checkext)
+	c.report()
     except KeyboardInterrupt:
 	if verbose > 0:
 	    print "[report interrupted]"
@@ -229,33 +225,37 @@ def main():
 
 class Checker:
 
-    def __init__(self):
+    def __init__(self, checkext=1):
+	self.reset()
+	self.checkext = checkext
+
+    def reset(self):
 	self.roots = []
 	self.todo = {}
 	self.done = {}
-	self.ext = {}
 	self.bad = {}
 	self.round = 0
 	# The following are not pickled:
 	self.robots = {}
+	self.errors = {}
 	self.urlopener = MyURLopener()
 	self.changed = 0
 
     def __getstate__(self):
-	return (self.roots, self.todo, self.done,
-		self.ext, self.bad, self.round)
+	return (self.roots, self.todo, self.done, self.bad, self.round)
 
     def __setstate__(self, state):
-	(self.roots, self.todo, self.done,
-	 self.ext, self.bad, self.round) = state
+	(self.roots, self.todo, self.done, self.bad, self.round) = state
 	for root in self.roots:
 	    self.addrobot(root)
+	for url in self.bad.keys():
+	    self.markerror(url)
 
     def addroot(self, root):
 	if root not in self.roots:
 	    self.roots.append(root)
 	    self.addrobot(root)
-	    self.newintlink(root, ("<root>", root))
+	    self.newlink(root, ("<root>", root))
 
     def addrobot(self, root):
 	url = urlparse.urljoin(root, "/robots.txt")
@@ -275,65 +275,25 @@ class Checker:
 	    self.round = self.round + 1
 	    if verbose > 0:
 		print
-		print "Round", self.round, self.status()
+		print "Round %d (%s)" % (self.round, self.status())
 		print 
 	    urls = self.todo.keys()[:roundsize]
 	    for url in urls:
 		self.dopage(url)
 
     def status(self):
-	return "(%d total, %d to do, %d done, %d external, %d bad)" % (
+	return "%d total, %d to do, %d done, %d bad" % (
 	    len(self.todo)+len(self.done),
 	    len(self.todo), len(self.done),
-	    len(self.ext), len(self.bad))
+	    len(self.bad))
 
-    def report(self, checkext=0):
+    def report(self):
 	print
 	if not self.todo: print "Final",
 	else: print "Interim",
-	print "Report", self.status()
-	if verbose > 0 or checkext:
-	    self.report_extrefs(checkext)
-	# Report errors last because the output may get truncated
+	print "Report (%s)" % self.status()
 	self.report_errors()
 
-    def report_extrefs(self, checkext=0):
-	if not self.ext:
-	    if verbose > 0:
-		print
-		print "No external URLs"
-	    return
-	if verbose > 0:
-	    print
-	    if checkext:
-		print "External URLs (checking validity):"
-	    else:
-		print "External URLs (not checked):"
-	    print
-	urls = self.ext.keys()
-	urls.sort()
-	for url in urls:
-	    if verbose > 0:
-		show("HREF ", url, " from", self.ext[url])
-	    if checkext:
-		self.checkextpage(url)
-
-    def checkextpage(self, url):
-	if url[:7] == 'mailto:' or url[:5] == 'news:':
-	    if verbose > 2: print "Not checking", url
-	    return
-	if verbose > 2: print "Checking", url, "..."
-	try:
-	    f = self.urlopener.open(url)
-	    safeclose(f)
-	    if verbose > 3: print "OK"
-	    if self.bad.has_key(url):
-		self.setgood(url)
-	except IOError, msg:
-	    msg = sanitize(msg)
-	    if verbose > 0: print "Error", msg
-	    self.setbad(url, msg)
-
     def report_errors(self):
 	if not self.bad:
 	    print
@@ -341,27 +301,10 @@ class Checker:
 	    return
 	print
 	print "Error Report:"
-	urls = self.bad.keys()
-	urls.sort()
-	bysource = {}
-	for url in urls:
-	    try:
-		origins = self.done[url]
-	    except KeyError:
-		try:
-		    origins = self.todo[url]
-		except KeyError:
-		    origins = self.ext[url]
-	    for source, rawlink in origins:
-		triple = url, rawlink, self.bad[url]
-		try:
-		    bysource[source].append(triple)
-		except KeyError:
-		    bysource[source] = [triple]
-	sources = bysource.keys()
+	sources = self.errors.keys()
 	sources.sort()
 	for source in sources:
-	    triples = bysource[source]
+	    triples = self.errors[source]
 	    print
 	    if len(triples) > 1:
 		print len(triples), "Errors in", source
@@ -376,31 +319,18 @@ class Checker:
     def dopage(self, url):
 	if verbose > 1:
 	    if verbose > 2:
-		show("Page  ", url, "  from", self.todo[url])
+		show("Check ", url, "  from", self.todo[url])
 	    else:
-		print "Page  ", url
+		print "Check ", url
 	page = self.getpage(url)
 	if page:
 	    for info in page.getlinkinfos():
 		link, rawlink = info
 		origin = url, rawlink
-		if not self.inroots(link):
-		    self.newextlink(link, origin)
-		else:
-		    self.newintlink(link, origin)
+		self.newlink(link, origin)
 	self.markdone(url)
 
-    def newextlink(self, url, origin):
-	try:
-	    self.ext[url].append(origin)
-	    if verbose > 3:
-		print "  New ext link", url
-	except KeyError:
-	    self.ext[url] = [origin]
-	    if verbose > 3:
-		print "  Seen ext link", url
-
-    def newintlink(self, url, origin):
+    def newlink(self, url, origin):
 	if self.done.has_key(url):
 	    self.newdonelink(url, origin)
 	else:
@@ -433,6 +363,13 @@ class Checker:
 	return 0
 
     def getpage(self, url):
+	if url[:7] == 'mailto:' or url[:5] == 'news:':
+	    if verbose > 1: print " Not checking mailto/news URL"
+	    return None
+	isint = self.inroots(url)
+	if not isint and not self.checkext:
+	    if verbose > 1: print " Not checking ext link"
+	    return None
 	try:
 	    f = self.urlopener.open(url)
 	except IOError, msg:
@@ -443,6 +380,10 @@ class Checker:
 		show(" HREF ", url, "  from", self.todo[url])
 	    self.setbad(url, msg)
 	    return None
+	if not isint:
+	    if verbose > 1: print " Not gathering links from ext URL"
+	    safeclose(f)
+	    return None
 	nurl = f.geturl()
 	info = f.info()
 	if info.has_key('content-type'):
@@ -477,6 +418,22 @@ class Checker:
 	    return
 	self.bad[url] = msg
 	self.changed = 1
+	self.markerror(url)
+	
+    def markerror(self, url):
+	try:
+	    origins = self.todo[url]
+	except KeyError:
+	    origins = self.done[url]
+	for source, rawlink in origins:
+	    triple = url, rawlink, self.bad[url]
+	    self.seterror(source, triple)
+
+    def seterror(self, url, triple):
+	try:
+	    self.errors[url].append(triple)
+	except KeyError:
+	    self.errors[url] = [triple]
 
 
 class Page: