Towards a standard access mechanism for URLs.

Lib/gopherlib.py

+# Gopher protocol client interface
+
+import string
+
+# Default selector, host and port
+DEF_SELECTOR = '1/'
+DEF_HOST     = 'gopher.micro.umn.edu'
+DEF_PORT     = 70
+
+# Recognized file types
+A_TEXT       = '0'
+A_MENU       = '1'
+A_CSO        = '2'
+A_ERROR      = '3'
+A_MACBINHEX  = '4'
+A_PCBINHEX   = '5'
+A_UUENCODED  = '6'
+A_INDEX      = '7'
+A_TELNET     = '8'
+A_BINARY     = '9'
+A_DUPLICATE  = '+'
+A_SOUND      = 's'
+A_EVENT      = 'e'
+A_CALENDAR   = 'c'
+A_HTML       = 'h'
+A_TN3270     = 'T'
+A_MIME       = 'M'
+A_IMAGE      = 'I'
+A_WHOIS      = 'w'
+A_QUERY      = 'q'
+A_GIF        = 'g'
+A_HTML       = 'h'			# HTML file
+A_WWW        = 'w'			# WWW address
+A_PLUS_IMAGE = ':'
+A_PLUS_MOVIE = ';'
+A_PLUS_SOUND = '<'
+
+
+# Function mapping all file types to strings; unknown types become TYPE='x'
+_names = dir()
+_type_to_name_map = None
+def type_to_name(gtype):
+	global _type_to_name_map
+	if not _type_to_name_map:
+		for name in _names:
+			if name[:2] == 'A_':
+				_type_to_name_map[eval(name)] = name[2:]
+	if _type_to_name_map.has_key(gtype):
+		return _type_to_name_map[gtype]
+	return 'TYPE=' + `gtype`
+
+# Names for characters and strings
+CRLF = '\r\n'
+TAB = '\t'
+
+# Send a selector to a given host and port, return a file with the reply
+def send_selector(selector, host, *args):
+	import socket
+	import string
+	if args:
+		if args[1:]: raise TypeError, 'too many args'
+		port = args[0]
+	else:
+		port = None
+		i = string.find(host, ':')
+		if i >= 0:
+			host, port = host[:i], string.atoi(host[i+1:])
+	if not port:
+		port = DEF_PORT
+	elif type(port) == type(''):
+		port = string.atoi(port)
+	s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+	s.connect(host, port)
+	s.send(selector + CRLF)
+	s.shutdown(1)
+	return s.makefile('r')
+
+# Send a selector and a query string
+def send_query(selector, query, host, *args):
+	return apply(send_selector, (selector + '\t' + query, host) + args)
+
+# The following functions interpret the data returned by the gopher
+# server according to the expected type, e.g. textfile or directory
+
+# Get a directory in the form of a list of entries
+def get_directory(f):
+	import string
+	list = []
+	while 1:
+		line = f.readline()
+		if not line:
+			print '(Unexpected EOF from server)'
+			break
+		if line[-2:] == CRLF:
+			line = line[:-2]
+		elif line[-1:] in CRLF:
+			line = line[:-1]
+		if line == '.':
+			break
+		if not line:
+			print '(Empty line from server)'
+			continue
+		gtype = line[0]
+		parts = string.splitfields(line[1:], TAB)
+		if len(parts) < 4:
+			print '(Bad line from server:', `line`, ')'
+			continue
+		if len(parts) > 4:
+			if parts[4:] != ['+']:
+			    print '(Extra info from server:', parts[4:], ')'
+		else:
+			parts.append('')
+		parts.insert(0, gtype)
+		list.append(parts)
+	return list
+
+# Get a text file as a list of lines, with trailing CRLF stripped
+def get_textfile(f):
+	list = []
+	get_alt_textfile(f, list.append)
+	return list
+
+# Get a text file and pass each line to a function, with trailing CRLF stripped
+def get_alt_textfile(f, func):
+	while 1:
+		line = f.readline()
+		if not line:
+			print '(Unexpected EOF from server)'
+			break
+		if line[-2:] == CRLF:
+			line = line[:-2]
+		elif line[-1:] in CRLF:
+			line = line[:-1]
+		if line == '.':
+			break
+		if line[:2] == '..':
+			line = line[1:]
+		func(line)
+
+# Get a binary file as one solid data block
+def get_binary(f):
+	data = f.read()
+	return data
+
+# Get a binary file and pass each block to a function
+def get_alt_binary(f, func, blocksize):
+	while 1:
+		data = f.read(blocksize)
+		if not data:
+			break
+		func(data)
+
+# Trivial test program
+def test():
+	import sys
+	import getopt
+	opts, args = getopt.getopt(sys.argv[1:], '')
+	selector = DEF_SELECTOR
+	type = selector[0]
+	host = DEF_HOST
+	port = DEF_PORT
+	if args:
+		host = args[0]
+		args = args[1:]
+	if args:
+		type = args[0]
+		args = args[1:]
+		if len(type) > 1:
+			type, selector = type[0], type
+		else:
+			selector = ''
+			if args:
+				selector = args[0]
+				args = args[1:]
+		query = ''
+		if args:
+			query = args[0]
+			args = args[1:]
+	if type == A_INDEX:
+		f = send_query(selector, query, host)
+	else:
+		f = send_selector(selector, host)
+	if type == A_TEXT:
+		list = get_textfile(f)
+		for item in list: print item
+	elif type in (A_MENU, A_INDEX):
+		list = get_directory(f)
+		for item in list: print item
+	else:
+		data = get_binary(f)
+		print 'binary data:', len(data), 'bytes:', `data[:100]`[:40]
+
+# Run the test when run as script
+if __name__ == '__main__':
+	test()

Lib/httplib.py

+# HTTP client class
+#
+# See the following document for a tentative protocol description:
+#     Hypertext Transfer Protocol (HTTP)        Tim Berners-Lee, CERN
+#     Internet Draft                                       5 Nov 1993
+#     draft-ietf-iiir-http-00.txt                  Expires 5 May 1994
+#
+# Example:
+#
+# >>> from httplib import HTTP
+# >>> h = HTTP('www.cwi.nl')
+# >>> h.putreqest('GET', '/index.html')
+# >>> h.putheader('Accept', 'text/html')
+# >>> h.putheader('Accept', 'text/plain')
+# >>> errcode, errmsg, headers = h.getreply()
+# >>> if errcode == 200:
+# ...     f = h.getfile()
+# ...     print f.read() # Print the raw HTML
+# ...
+# <TITLE>Home Page of CWI, Amsterdam</TITLE>
+# [...many more lines...]
+# >>>
+#
+# Note that an HTTP object is used for a single request -- to issue a
+# second request to the same server, you create a new HTTP object.
+# (This is in accordance with the protocol, which uses a new TCP
+# connection for each request.)
+
+
+import os
+import socket
+import string
+import regex
+import regsub
+import rfc822
+
+HTTP_VERSION = 'HTTP/1.0'
+HTTP_PORT = 80
+
+replypat = regsub.gsub('\\.', '\\\\.', HTTP_VERSION) + \
+	  '[ \t]+\([0-9][0-9][0-9]\)\(.*\)'
+replyprog = regex.compile(replypat)
+
+class HTTP:
+
+	def __init__(self, *args):
+		self.debuglevel = 0
+		if args: apply(self.connect, args)
+
+	def set_debuglevel(self, debuglevel):
+		self.debuglevel = debuglevel
+
+	def connect(self, host, *args):
+		if args:
+			if args[1:]: raise TypeError, 'too many args'
+			port = args[0]
+		else:
+			i = string.find(host, ':')
+			if i >= 0:
+				host, port = host[:i], host[i+1:]
+				try: port = string.atoi(port)
+				except string.atoi_error: port = None
+		if not port: port = HTTP_PORT
+		self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+		if self.debuglevel > 0: print 'connect:', (host, port)
+		self.sock.connect(host, port)
+
+	def send(self, str):
+		if self.debuglevel > 0: print 'send:', `str`
+		self.sock.send(str)
+
+	def putrequest(self, request, selector):
+		str = '%s %s %s\r\n' % (request, selector, HTTP_VERSION)
+		self.send(str)
+
+	def putheader(self, header, *args):
+		str = '%s: %s\r\n' % (header, string.joinfields(args,'\r\n\t'))
+		self.send(str)
+
+	def endheaders(self):
+		self.send('\r\n')
+
+	def endrequest(self):
+		if self.debuglevel > 0: print 'shutdown: 1'
+		self.sock.shutdown(1)
+
+	def getreply(self):
+		self.endrequest()
+		self.file = self.sock.makefile('r')
+		line = self.file.readline()
+		if self.debuglevel > 0: print 'reply:', `line`
+		if replyprog.match(line) < 0:
+			self.headers = None
+			return -1, line, self.headers
+		errcode, errmsg = replyprog.group(1, 2)
+		errcode = string.atoi(errcode)
+		errmsg = string.strip(errmsg)
+		self.headers = rfc822.Message(self.file)
+		return errcode, errmsg, self.headers
+
+	def getfile(self):
+		return self.file
+
+
+def test():
+	import sys
+	import getopt
+	opts, args = getopt.getopt(sys.argv[1:], 'd')
+	dl = 0
+	for o, a in opts:
+		if o == '-d': dl = dl + 1
+	host = 'www.cwi.nl:80'
+	selector = '/index.html'
+	if args[0:]: host = args[0]
+	if args[1:]: selector = args[1]
+	h = HTTP()
+	h.set_debuglevel(dl)
+	h.connect(host)
+	h.putrequest('GET', selector)
+	errcode, errmsg, headers = h.getreply()
+	print 'errcode =', errcode
+	print 'headers =', headers
+	print 'errmsg  =', errmsg
+	if headers:
+		for header in headers.headers: print string.strip(header)
+	print h.getfile().read()
+
+if __name__ == '__main__':
+	test()

Lib/urlopen.py

+# Open an arbitrary URL
+#
+# See the following document for a tentative description of URLs:
+#     Uniform Resource Locators              Tim Berners-Lee
+#     INTERNET DRAFT                                    CERN
+#     IETF URL Working Group                    14 July 1993
+#     draft-ietf-uri-url-01.txt
+#
+# The object returned by urlopen() will differ per protocol.
+# All you know is that is has methods read(), fileno(), close() and info().
+# The read(), fileno() and close() methods work like those of open files.
+# The info() method returns an rfc822.Message object which can be
+# used to query various info about the object, if available.
+# (rfc822.Message objects are queried with the getheader() method.)
+
+import socket
+import regex
+import regsub
+import string
+import rfc822
+import ftplib
+
+
+# External interface -- use urlopen(file) as if it were open(file, 'r')
+def urlopen(url):
+	url = string.strip(url)
+	if url[:1] == '<' and url[-1:] == '>': url = string.strip(url[1:-1])
+	if url[:4] == 'URL:': url = string.strip(url[4:])
+	type, url = splittype(url)
+	if not type: type = 'file'
+	type = regsub.gsub('-', '_', type)
+	try:
+		func = eval('open_' + type)
+	except NameError:
+		raise IOError, ('url error', 'unknown url type', type)
+	try:
+		return func(url)
+	except socket.error, msg:
+		raise IOError, ('socket error', msg)
+
+
+# Each routine of the form open_<type> knows how to open that type of URL
+
+# Use HTTP protocol
+def open_http(url):
+	import httplib
+	host, selector = splithost(url)
+	h = httplib.HTTP(host)
+	h.putrequest('GET', selector)
+	errcode, errmsg, headers = h.getreply()
+	if errcode == 200: return makefile(h.getfile(), headers)
+	else: raise IOError, ('http error', errcode, errmsg, headers)
+
+# Empty rfc822.Message object
+noheaders = rfc822.Message(open('/dev/null', 'r'))
+noheaders.fp.close()			# Recycle file descriptor
+
+# Use Gopher protocol
+def open_gopher(url):
+	import gopherlib
+	host, selector = splithost(url)
+	type, selector = splitgophertype(selector)
+	selector, query = splitquery(selector)
+	if query: fp = gopherlib.send_query(selector, query, host)
+	else: fp = gopherlib.send_selector(selector, host)
+	return makefile(fp, noheaders)
+
+# Use local file or FTP depending on form of URL
+localhost = socket.gethostbyname('localhost')
+thishost = socket.gethostbyname(socket.gethostname())
+def open_file(url):
+	host, file = splithost(url)
+	if not host: return makefile(open(file, 'r'), noheaders)
+	host, port = splitport(host)
+	if not port and socket.gethostbyname(host) in (localhost, thishost):
+		try: fp = open(file, 'r')
+		except IOError: fp = None
+		if fp: return makefile(fp, noheaders)
+	return open_ftp(url)
+
+# Use FTP protocol
+ftpcache = {}
+ftperrors = (ftplib.error_reply,
+	     ftplib.error_temp,
+	     ftplib.error_perm,
+	     ftplib.error_proto)
+def open_ftp(url):
+	host, file = splithost(url)
+	host, port = splitport(host)
+	host = socket.gethostbyname(host)
+	if not port: port = ftplib.FTP_PORT
+	key = (host, port)
+	try:
+		if not ftpcache.has_key(key):
+			ftpcache[key] = ftpwrapper(host, port)
+		return makefile(ftpcache[key].retrfile(file), noheaders)
+	except ftperrors, msg:
+		raise IOError, ('ftp error', msg)
+
+
+# Utility classes
+
+# Class used to add an info() method to a file object
+class makefile:
+	def __init__(self, fp, headers):
+		self.fp = fp
+		self.headers = headers
+		self.read = self.fp.read
+		self.fileno = self.fp.fileno
+		self.close = self.fp.close
+	def info(self):
+		return self.headers
+
+# Class used by open_ftp() for cache of open FTP connections
+class ftpwrapper:
+	def __init__(self, host, port):
+		self.host = host
+		self.port = port
+		self.init()
+	def init(self):
+		self.ftp = ftplib.FTP()
+		self.ftp.connect(self.host, self.port)
+		self.ftp.login()
+	def retrfile(self, file):
+		try:
+			self.ftp.voidcmd('TYPE I')
+		except ftplib.all_errors:
+			self.init()
+			self.ftp.voidcmd('TYPE I')
+		conn = None
+		if file:
+			try:
+				cmd = 'RETR ' + file
+				conn = self.ftp.transfercmd(cmd)
+			except ftplib.error_perm, reason:
+				if reason[:3] != '550':
+					raise IOError, ('ftp error', reason)
+		if not conn:
+			# Try a directory listing
+			if file: cmd = 'NLST ' + file
+			else: cmd = 'NLST'
+			conn = self.ftp.transfercmd(cmd)
+		return fakefile(self.ftp, conn)
+
+# Class used by ftpwrapper to handle response when transfer is complete
+class fakefile:
+	def __init__(self, ftp, conn):
+		self.ftp = ftp
+		self.conn = conn
+		self.fp = self.conn.makefile('r')
+		self.read = self.fp.read
+		self.fileno = self.fp.fileno
+	def __del__(self):
+		self.close()
+	def close(self):
+		self.conn = None
+		self.fp = None
+		self.read = None
+		if self.ftp: self.ftp.voidresp()
+		self.ftp = None
+
+
+# Utilities to split url parts into components:
+# splittype('type:opaquestring') --> 'type', 'opaquestring'
+# splithost('//host[:port]/path') --> 'host[:port]', '/path'
+# splitport('host:port') --> 'host', 'port'
+# splitquery('/path?query') --> '/path', 'query'
+# splittag('/path#tag') --> '/path', 'tag'
+# splitgophertype('/Xselector') --> 'X', 'selector'
+
+typeprog = regex.compile('^\([^/:]+\):\(.*\)$')
+def splittype(url):
+	if typeprog.match(url) >= 0: return typeprog.group(1, 2)
+	return None, url
+
+hostprog = regex.compile('^//\([^/]+\)\(.*\)$')
+def splithost(url):
+	if hostprog.match(url) >= 0: return hostprog.group(1, 2)
+	return None, url
+
+portprog = regex.compile('^\(.*\):\([0-9]+\)$')
+def splitport(host):
+	if portprog.match(host) >= 0: return portprog.group(1, 2)
+	return host, None
+
+queryprog = regex.compile('^\(.*\)\?\([^?]*\)$')
+def splitquery(url):
+	if queryprog.match(url) >= 0: return queryprog.group(1, 2)
+	return url, None
+
+tagprog = regex.compile('^\(.*\)#\([^#]*\)$')
+def splittag(url):
+	if tagprog.match(url) >= 0: return tagprog.group(1, 2)
+	return url, None
+
+def splitgophertype(selector):
+	if selector[:1] == '/' and selector[1:2]:
+		return selector[1], selector[2:]
+	return None, selector
+
+
+# Test program
+def test():
+	import sys
+	args = sys.argv[1:]
+	if not args:
+		args = [
+			'/etc/passwd',
+			'file:/etc/passwd',
+			'file://localhost/etc/passwd',
+			'ftp://ftp.cwi.nl/etc/passwd',
+			'gopher://gopher.cwi.nl/11/',
+			'http://www.cwi.nl/index.html',
+			]
+	for arg in args:
+		print '-'*10, arg, '-'*10
+		print regsub.gsub('\r', '', urlopen(arg).read())
+	print '-'*40
+
+# Run test program when run as a script
+if __name__ == '__main__':
+	test()