Commit a8f22b58 authored by Takeshi Yamamoto's avatar Takeshi Yamamoto

added a hook for CJK process

parent 203c23b4
......@@ -80,7 +80,8 @@ class Lexicon(Persistent):
def termToWordIds(self, text):
last = _text2list(text)
for element in self._pipeline:
last = element.process(last)
process = getattr(element, "process_post_glob", element.process)
last = process(last)
wids = []
for word in last:
wids.append(self._wids.get(word, 0))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment