Charles G. Waldman <cgw@fnal.gov>:

Add the EXTENDED_ARG opcode to the virtual machine, allowing 32-bit
arguments to opcodes instead of being forced to stick to the 16-bit
limit.  This is especially useful for machine-generated code, which
can be too long for the SET_LINENO parameter to fit into 16 bits.

This closes the implementation portion of SourceForge patch #100893.

Include/node.h

@@ -19,8 +19,8 @@ extern "C" {
 typedef struct _node {
     short		n_type;
     char		*n_str;
-    short		n_lineno;
-    short		n_nchildren;
+    int			n_lineno;
+    int			n_nchildren;
     struct _node	*n_child;
 } node;
 

Include/opcode.h

@@ -130,6 +130,9 @@ redistribution of this file, and for a DISCLAIMER OF ALL WARRANTIES.
 #define CALL_FUNCTION_KW           141	/* #args + (#kwargs<<8) */
 #define CALL_FUNCTION_VAR_KW       142	/* #args + (#kwargs<<8) */
 
+/* Support for opargs more than 16 bits long */
+#define EXTENDED_ARG  143
+
 /* Comparison operator codes (argument to COMPARE_OP) */
 enum cmp_op {LT, LE, EQ, NE, GT, GE, IN, NOT_IN, IS, IS_NOT, EXC_MATCH, BAD};
 

Lib/dis.py

@@ -56,6 +56,7 @@ def disassemble(co, lasti=-1):
 	labels = findlabels(code)
 	n = len(code)
 	i = 0
+	extended_arg = 0
 	while i < n:
 		c = code[i]
 		op = ord(c)
@@ -68,8 +69,11 @@ def disassemble(co, lasti=-1):
 		print string.ljust(opname[op], 20),
 		i = i+1
 		if op >= HAVE_ARGUMENT:
-			oparg = ord(code[i]) + ord(code[i+1])*256
+			oparg = ord(code[i]) + ord(code[i+1])*256 + extended_arg
+			extended_arg = 0
 			i = i+2
+			if op == EXTENDED_ARG:
+				extended_arg = oparg*65536L
 			print string.rjust(`oparg`, 5),
 			if op in hasconst:
 				print '(' + `co.co_consts[oparg]` + ')',
@@ -258,6 +262,8 @@ def_op('CALL_FUNCTION_VAR', 140)     # #args + (#kwargs << 8)
 def_op('CALL_FUNCTION_KW', 141)      # #args + (#kwargs << 8)
 def_op('CALL_FUNCTION_VAR_KW', 142)  # #args + (#kwargs << 8)
 
+def_op('EXTENDED_ARG', 143) 
+EXTENDED_ARG = 143
 
 def _test():
 	"""Simple test program to disassemble a file."""

Parser/node.c

@@ -8,7 +8,12 @@ See the file "Misc/COPYRIGHT" for information on usage and
 redistribution of this file, and for a DISCLAIMER OF ALL WARRANTIES.
 ******************************************************************/
 
+#ifdef HAVE_LIMITS_H
 #include <limits.h>
+#endif
+#ifndef INT_MAX
+#define INT_MAX 2147483647
+#endif
 
 /* Parse tree node implementation */
 
@@ -39,7 +44,7 @@ PyNode_AddChild(register node *n1, int type, char *str, int lineno)
 	register int nch = n1->n_nchildren;
 	register int nch1 = nch+1;
 	register node *n;
-	if (nch == SHRT_MAX || nch < 0)
+	if (nch == INT_MAX || nch < 0)
 		return E_OVERFLOW;
 	if (XXXROUNDUP(nch) < nch1) {
 		n = n1->n_child;

Python/ceval.c

@@ -613,6 +613,7 @@ eval_code2(PyCodeObject *co, PyObject *globals, PyObject *locals,
 		opcode = NEXTOP();
 		if (HAS_ARG(opcode))
 			oparg = NEXTARG();
+	  dispatch_opcode:
 #ifdef DYNAMIC_EXECUTION_PROFILE
 #ifdef DXPAIRS
 		dxpairs[lastopcode][opcode]++;
@@ -1750,6 +1751,11 @@ eval_code2(PyCodeObject *co, PyObject *globals, PyObject *locals,
 			if (x != NULL) continue;
 			break;
 
+		case EXTENDED_ARG:
+			opcode = NEXTOP();
+			oparg = oparg<<16 | NEXTARG();
+			goto dispatch_opcode;
+			break;
 
 		default:
 			fprintf(stderr,

Python/compile.c

@@ -36,6 +36,9 @@ redistribution of this file, and for a DISCLAIMER OF ALL WARRANTIES.
 #ifdef HAVE_LIMITS_H
 #include <limits.h>
 #endif
+#ifndef INT_MAX
+#define INT_MAX 2147483647
+#endif
 
 /* Three symbols from graminit.h are also defined in Python.h, with
    Py_ prefixes to their names.  Python.h can't include graminit.h
@@ -572,11 +575,17 @@ com_set_lineno(struct compiling *c, int lineno)
 static void
 com_addoparg(struct compiling *c, int op, int arg)
 {
+	int extended_arg = arg >> 16;
 	if (op == SET_LINENO) {
 		com_set_lineno(c, arg);
 		if (Py_OptimizeFlag)
 			return;
 	}
+	if (extended_arg){
+		com_addbyte(c, EXTENDED_ARG);
+		com_addint(c, extended_arg);
+		arg &= 0xffff;
+	}
 	com_addbyte(c, op);
 	com_addint(c, arg);
 }
@@ -606,7 +615,14 @@ com_backpatch(struct compiling *c, int anchor)
 		prev = code[anchor] + (code[anchor+1] << 8);
 		dist = target - (anchor+2);
 		code[anchor] = dist & 0xff;
-		code[anchor+1] = dist >> 8;
+		dist >>= 8;
+		code[anchor+1] = dist;
+		dist >>= 8;
+		if (dist) {
+			com_error(c, PyExc_SystemError,
+				  "com_backpatch: offset too large");
+			break;
+		}
 		if (!prev)
 			break;
 		anchor -= prev;
@@ -3364,6 +3380,7 @@ optimize(struct compiling *c)
 			break;
 		if (HAS_ARG(opcode))
 			oparg = NEXTARG();
+	  dispatch_opcode1:
 		switch (opcode) {
 		case STORE_NAME:
 		case DELETE_NAME:
@@ -3374,6 +3391,11 @@ optimize(struct compiling *c)
 		case EXEC_STMT:
 			c->c_flags &= ~CO_OPTIMIZED;
 			break;
+		case EXTENDED_ARG:
+			opcode = NEXTOP();
+			oparg = oparg<<16 | NEXTARG();
+			goto dispatch_opcode1;
+			break;
 		}
 	}
 	
@@ -3389,6 +3411,7 @@ optimize(struct compiling *c)
 			break;
 		if (HAS_ARG(opcode))
 			oparg = NEXTARG();
+	  dispatch_opcode2:
 		if (opcode == LOAD_NAME ||
 		    opcode == STORE_NAME ||
 		    opcode == DELETE_NAME) {
@@ -3403,13 +3426,20 @@ optimize(struct compiling *c)
 				continue;
 			}
 			i = PyInt_AsLong(v);
+			if (i >> 16) /* too big for 2 bytes */
+				continue;
 			switch (opcode) {
 			case LOAD_NAME: cur_instr[0] = LOAD_FAST; break;
 			case STORE_NAME: cur_instr[0] = STORE_FAST; break;
 			case DELETE_NAME: cur_instr[0] = DELETE_FAST; break;
 			}
 			cur_instr[1] = i & 0xff;
-			cur_instr[2] = (i>>8) & 0xff;
+			cur_instr[2] = i >> 8;
+		}
+		if (opcode == EXTENDED_ARG) {
+			opcode = NEXTOP();
+			oparg = oparg<<16 | NEXTARG();
+			goto dispatch_opcode2;
 		}
 	}