Commit 239bf895 authored by Stefan Behnel's avatar Stefan Behnel

remove legacy pickling of Lexicon.py

parent ae3f3f4e
...@@ -2,9 +2,6 @@ ...@@ -2,9 +2,6 @@
# Pyrex Scanner # Pyrex Scanner
# #
#import pickle
import cPickle as pickle
import os import os
import platform import platform
import stat import stat
...@@ -33,122 +30,13 @@ debug_scanner = 0 ...@@ -33,122 +30,13 @@ debug_scanner = 0
trace_scanner = 0 trace_scanner = 0
scanner_debug_flags = 0 scanner_debug_flags = 0
scanner_dump_file = None scanner_dump_file = None
binary_lexicon_pickle = 1
notify_lexicon_unpickling = 0
notify_lexicon_pickling = 1
lexicon = None lexicon = None
#-----------------------------------------------------------------
def hash_source_file(path):
# Try to calculate a hash code for the given source file.
# Returns an empty string if the file cannot be accessed.
#print "Hashing", path ###
try:
from hashlib import md5 as new_md5
except ImportError:
from md5 import new as new_md5
f = None
try:
try:
f = open(path, "rU")
text = f.read()
except IOError, e:
print("Unable to hash scanner source file (%s)" % e)
return ""
finally:
if f:
f.close()
# Normalise spaces/tabs. We don't know what sort of
# space-tab substitution the file may have been
# through, so we replace all spans of spaces and
# tabs by a single space.
import re
text = re.sub("[ \t]+", " ", text)
hash = new_md5(text.encode("ASCII")).hexdigest()
return hash
def open_pickled_lexicon(expected_hash):
# Try to open pickled lexicon file and verify that
# it matches the source file. Returns the opened
# file if successful, otherwise None. ???
global lexicon_pickle
f = None
result = None
if os.path.exists(lexicon_pickle):
try:
f = open(lexicon_pickle, "rb")
actual_hash = pickle.load(f)
if actual_hash == expected_hash:
result = f
f = None
else:
print("Lexicon hash mismatch:") ###
print(" expected " + expected_hash) ###
print(" got " + actual_hash) ###
except (IOError, pickle.UnpicklingError), e:
print("Warning: Unable to read pickled lexicon " + lexicon_pickle)
print(e)
if f:
f.close()
return result
def try_to_unpickle_lexicon():
global lexicon, lexicon_pickle, lexicon_hash
dir = os.path.dirname(__file__)
source_file = os.path.join(dir, "Lexicon.py")
lexicon_hash = hash_source_file(source_file)
lexicon_pickle = os.path.join(dir, "Lexicon.pickle")
f = open_pickled_lexicon(lexicon_hash)
if f:
if notify_lexicon_unpickling:
t0 = time()
print("Unpickling lexicon...")
try:
lexicon = pickle.load(f)
except Exception, e:
print "WARNING: Exception while loading lexicon pickle, regenerating"
print e
lexicon = None
f.close()
if notify_lexicon_unpickling:
t1 = time()
print("Done (%.2f seconds)" % (t1 - t0))
def create_new_lexicon():
global lexicon
t0 = time()
print("Creating lexicon...")
lexicon = make_lexicon()
t1 = time()
print("Done (%.2f seconds)" % (t1 - t0))
def pickle_lexicon():
f = None
try:
f = open(lexicon_pickle, "wb")
except IOError:
print("Warning: Unable to save pickled lexicon in " + lexicon_pickle)
if f:
if notify_lexicon_pickling:
t0 = time()
print("Pickling lexicon...")
pickle.dump(lexicon_hash, f, binary_lexicon_pickle)
pickle.dump(lexicon, f, binary_lexicon_pickle)
f.close()
if notify_lexicon_pickling:
t1 = time()
print("Done (%.2f seconds)" % (t1 - t0))
def get_lexicon(): def get_lexicon():
global lexicon global lexicon
if not lexicon and plex_version is None:
try_to_unpickle_lexicon()
if not lexicon: if not lexicon:
create_new_lexicon() lexicon = make_lexicon()
if plex_version is None:
pickle_lexicon()
return lexicon return lexicon
#------------------------------------------------------------------ #------------------------------------------------------------------
......
...@@ -42,7 +42,6 @@ if sys.version_info < (2,4): ...@@ -42,7 +42,6 @@ if sys.version_info < (2,4):
cython_dir = os.path.join(get_python_lib(prefix=''), 'Cython') cython_dir = os.path.join(get_python_lib(prefix=''), 'Cython')
compiler_dir = os.path.join(cython_dir, 'Compiler') compiler_dir = os.path.join(cython_dir, 'Compiler')
setup_args['data_files'] = [ setup_args['data_files'] = [
(compiler_dir, ['Cython/Compiler/Lexicon.pickle']),
(cython_dir, [ f for pattern in (cython_dir, [ f for pattern in
['Cython/Includes/*.pxd', ['Cython/Includes/*.pxd',
'Cython/Plex/*.pxd', 'Cython/Plex/*.pxd',
...@@ -50,8 +49,7 @@ if sys.version_info < (2,4): ...@@ -50,8 +49,7 @@ if sys.version_info < (2,4):
'Cython/Runtime/*.pyx'] 'Cython/Runtime/*.pyx']
for f in glob.glob(pattern) ])] for f in glob.glob(pattern) ])]
else: else:
setup_args['package_data'] = {'Cython.Compiler' : ['Lexicon.pickle'], setup_args['package_data'] = {'Cython' : ['Includes/*.pxd',
'Cython' : ['Includes/*.pxd',
'Plex/*.pxd', 'Plex/*.pxd',
'Compiler/*.pxd', 'Compiler/*.pxd',
'Runtime/*.pyx']} 'Runtime/*.pyx']}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment