Add some parsing tests

- byte order marker
- newline between decorator and its function
- strings with size >64k [including a fix]

also, a decimal.Decimal test

src/codegen/parse_ast.py

@@ -4,8 +4,8 @@ import sys
 from types import NoneType
 
 def _print_str(s, f):
-    assert len(s) < 2**16
-    f.write(struct.pack(">H", len(s)))
+    assert len(s) < 2**32
+    f.write(struct.pack(">L", len(s)))
     f.write(s)
 
 TYPE_MAP = {

src/codegen/parser.cpp

@@ -111,7 +111,7 @@ AST_expr* readASTExpr(BufferedReader* reader);
 AST_stmt* readASTStmt(BufferedReader* reader);
 
 static std::string readString(BufferedReader* reader) {
-    int strlen = reader->readShort();
+    int strlen = reader->readUInt();
     llvm::SmallString<32> chars;
     for (int i = 0; i < strlen; i++) {
         chars.push_back(reader->readByte());
@@ -120,7 +120,7 @@ static std::string readString(BufferedReader* reader) {
 }
 
 InternedString BufferedReader::readAndInternString() {
-    int strlen = readShort();
+    int strlen = readUInt();
     llvm::SmallString<32> chars;
     for (int i = 0; i < strlen; i++) {
         chars.push_back(readByte());
@@ -1026,9 +1026,9 @@ AST_Module* parse_file(const char* fn) {
 
 const char* getMagic() {
     if (ENABLE_PYPA_PARSER)
-        return "a\ncL";
+        return "a\ncM";
     else
-        return "a\ncl";
+        return "a\ncm";
 }
 
 #define MAGIC_STRING_LENGTH 4

src/codegen/serialize_ast.cpp

@@ -38,19 +38,24 @@ private:
     SerializeASTVisitor(FILE* file) : file(file), checksum(0) {}
     virtual ~SerializeASTVisitor() {}
 
-    void writeByte(uint8_t v) {
-        fwrite(&v, 1, sizeof(v), file);
-        checksum ^= v;
+    void writeByte(uint64_t v) {
+        assert(v < 256);
+
+        uint8_t b = (uint8_t)v;
+        fwrite(&b, 1, 1, file);
+        checksum ^= b;
     }
 
-    void writeShort(uint16_t v) {
+    void writeShort(uint64_t v) {
+        RELEASE_ASSERT(v < (1 << 16), "");
         // I guess we use big-endian:
         for (int i = 1; i >= 0; i--) {
             writeByte((v >> (i * 8)) & 0xff);
         }
     }
 
-    void writeUInt(uint32_t v) {
+    void writeUInt(uint64_t v) {
+        RELEASE_ASSERT(v < (1L << 32), "");
         for (int i = 3; i >= 0; i--) {
             writeByte((v >> (i * 8)) & 0xff);
         }
@@ -71,7 +76,7 @@ private:
     }
 
     void writeString(const std::string& v) {
-        writeShort(v.size());
+        writeUInt(v.size());
         fwrite(v.c_str(), 1, v.size(), file);
         for (int i = 0; i < v.size(); i++) {
             checksum ^= v[i];

test/tests/decimal_test.py

+# expected: fail
+
+from decimal import Decimal
+
+for d in (Decimal("0.5"), Decimal("0"), Decimal(0), Decimal(1.0)):
+    for f in [0, 1, 0.5, 0.0, 0.1]:
+        print d, f,
+        print d < f, d <= f, d > f, d >= f, d == f, d != f

test/tests/decorator_whitespace.py

+# fail-if: '-x' not in EXTRA_JIT_ARGS
+
+def dec(f):
+    return f
+
+@dec
+
+def f():
+    pass

test/tests/parsing_bom.py

+# fail-if: '-x' not in EXTRA_JIT_ARGS
+
+# I really don't understand all the intricacies of unicode parsing, but apparently in addition to
+# Python-specific coding lines, you can put a unicode byte order mark to signify that the text
+# is encoded.
+# http://en.wikipedia.org/wiki/Byte_order_mark
+s = """\xef\xbb\xbfprint "hello world" """
+exec s

test/tests/super_long_string.py

+import os
+import sys
+
+s = 'def f():\n    return """' + '.' * 100000 + '"""'
+
+path = os.path.join(os.path.dirname(__file__), "super_long_string_target.py")
+
+with open(path, 'w') as f:
+    f.write(s)
+
+import super_long_string_target
+del sys.modules["super_long_string_target"]
+import super_long_string_target
+
+os.remove(path)
+os.remove(path + 'c')