Commit fd3a9291 authored by Tim Peters's avatar Tim Peters

Comment the special status of wid 0.

globToWordIds():  This was building a list of words and then throwing
it away without referencing it.  Deleted the code.
parent 92a2995e
......@@ -26,7 +26,11 @@ class Lexicon:
def __init__(self, *pipeline):
self._wids = OIBTree() # word -> wid
self._words = IOBTree() # wid -> word
# XXX we're reserving wid 0, but that might be yagni
# wid 0 is reserved for words that aren't in the lexicon (OOV -- out
# of vocabulary). This can happen, e.g., if a query contains a word
# we never saw before, and that isn't a known stopword (or otherwise
# filtered out). Returning a special wid value for OOV words is a
# way to let clients know when an OOV word appears.
self._nextwid = 1
self._pipeline = pipeline
......@@ -78,12 +82,10 @@ class Lexicon:
assert prefix and not prefix.endswith("*")
keys = self._wids.keys(prefix) # Keys starting at prefix
wids = []
words = []
for key in keys:
if not key.startswith(prefix):
break
wids.append(self._wids[key])
words.append(key)
return wids
def _getWordIdCreate(self, word):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment