Revise the markup related to the grammar productions to increase the

level of predictability.  This is not really "good" markup, but is arguably
better than we had before.
This closes SF bug #523117.

Doc/perl/python.perl

@@ -807,6 +807,18 @@ sub do_cmd_production{
             . $_);
 }
 
+sub do_cmd_productioncont{
+    local($_) = @_;
+    my $defn = next_argument();
+    return ("<tr valign=\"baseline\">\n"
+            . "    <td>&nbsp;</td>\n"
+            . "    <td>&nbsp;</td>\n"
+            . "    <td><code>"
+            . translate_commands($defn)
+            . "</code></td></tr>"
+            . $_);
+}
+
 sub process_grammar_files{
     my $lang;
     my $filename;
@@ -847,6 +859,7 @@ sub process_grammar_files{
 
 sub strip_grammar_markup{
     local($_) = @_;
+    s/\\productioncont/              /g;
     s/\\production(<<\d+>>)(.+)\1/\n\2 ::= /g;
     s/\\token(<<\d+>>)(.+)\1/\2/g;
     s/\\e([^a-zA-Z])/\\\1/g;

Doc/ref/ref2.tex

@@ -318,8 +318,8 @@ String literals are described by the following lexical definitions:
              {"'" \token{shortstringitem}* "'"
               | '"' \token{shortstringitem}* '"'}
   \production{longstring}
-             {"'''" \token{longstringitem}* "'''"
-              | '"""' \token{longstringitem}* '"""'}
+             {"'''" \token{longstringitem}* "'''"}
+  \productioncont{| '"""' \token{longstringitem}* '"""'}
   \production{shortstringitem}
              {\token{shortstringchar} | \token{escapeseq}}
   \production{longstringitem}

Doc/ref/ref5.tex

@@ -54,8 +54,8 @@ categorized syntactically as atoms.  The syntax for atoms is:
   \production{atom}
              {\token{identifier} | \token{literal} | \token{enclosure}}
   \production{enclosure}
-             {\token{parenth_form} | \token{list_display}
-              | \token{dict_display} | \token{string_conversion}}
+             {\token{parenth_form} | \token{list_display}}
+  \productioncont{| \token{dict_display} | \token{string_conversion}}
 \end{productionlist}
 
 
@@ -112,9 +112,8 @@ Python supports string literals and various numeric literals:
 
 \begin{productionlist}
   \production{literal}
-             {\token{stringliteral} | \token{integer}
-              | \token{longinteger} | \token{floatnumber}
-              | \token{imagnumber}}
+             {\token{stringliteral} | \token{integer} | \token{longinteger}}
+  \productioncont{| \token{floatnumber} | \token{imagnumber}}
 \end{productionlist}
 
 Evaluation of a literal yields an object of the given type (string,
@@ -439,13 +438,12 @@ series of arguments:
   \production{call}
              {\token{primary} "(" [\token{argument_list} [","]] ")"}
   \production{argument_list}
-             {\token{positional_arguments} ["," \token{keyword_arguments} 
-		 ["," "*" \token{expression} ["," "**" \token{expression}]]]
-              | \token{keyword_arguments} ["," "*" \token{expression} 
-		["," "**" \token{expression}]]
-	      | "*" \token{expression} ["," "**" \token{expression}]
-	      | "**" \token{expression}
-	     }
+             {\token{positional_arguments} ["," \token{keyword_arguments}}
+  \productioncont{   ["," "*" \token{expression} ["," "**" \token{expression}]]]}
+  \productioncont{| \token{keyword_arguments} ["," "*" \token{expression}}
+  \productioncont{                             ["," "**" \token{expression}]]}
+  \productioncont{| "*" \token{expression} ["," "**" \token{expression}]}
+  \productioncont{| "**" \token{expression}}
   \production{positional_arguments}
              {\token{expression} ("," \token{expression})*}
   \production{keyword_arguments}
@@ -666,8 +664,8 @@ operators:
 \begin{productionlist}
   \production{m_expr}
              {\token{u_expr} | \token{m_expr} "*" \token{u_expr}
-              | \token{m_expr} "/" \token{u_expr}
-              | \token{m_expr} "\%" \token{u_expr}}
+              | \token{m_expr} "/" \token{u_expr}}
+  \productioncont{| \token{m_expr} "\%" \token{u_expr}}
   \production{a_expr}
              {\token{m_expr} | \token{aexpr} "+" \token{m_expr}
               \token{aexpr} "-" \token{m_expr}}
@@ -801,8 +799,8 @@ interpretation that is conventional in mathematics:
   \production{comparison}
              {\token{or_expr} ( \token{comp_operator} \token{or_expr} )*}
   \production{comp_operator}
-             {"<" | ">" | "==" | ">=" | "<=" | "<>" | "!="
-              | "is" ["not"] | ["not"] "in"}
+             {"<" | ">" | "==" | ">=" | "<=" | "<>" | "!="}
+  \productioncont{| "is" ["not"] | ["not"] "in"}
 \end{productionlist}
 
 Comparisons yield integer values: \code{1} for true, \code{0} for false.

Doc/ref/ref6.tex

@@ -6,22 +6,21 @@ Several simple statements may occur on a single line separated
 by semicolons.  The syntax for simple statements is:
 
 \begin{productionlist}
-  \production{simple_stmt}
-             {\token{expression_stmt}
-              | \token{assert_stmt}
-              | \token{assignment_stmt}
-              | \token{augmented_assignment_stmt}
-              | \token{pass_stmt}
-              | \token{del_stmt}
-              | \token{print_stmt}
-              | \token{return_stmt}
-              | \token{yield_stmt}
-              | \token{raise_stmt}
-              | \token{break_stmt}
-              | \token{continue_stmt}
-              | \token{import_stmt}
-              | \token{global_stmt}
-              | \token{exec_stmt}}
+  \production{simple_stmt}{\token{expression_stmt}}
+  \productioncont{| \token{assert_stmt}}
+  \productioncont{| \token{assignment_stmt}}
+  \productioncont{| \token{augmented_assignment_stmt}}
+  \productioncont{| \token{pass_stmt}}
+  \productioncont{| \token{del_stmt}}
+  \productioncont{| \token{print_stmt}}
+  \productioncont{| \token{return_stmt}}
+  \productioncont{| \token{yield_stmt}}
+  \productioncont{| \token{raise_stmt}}
+  \productioncont{| \token{break_stmt}}
+  \productioncont{| \token{continue_stmt}}
+  \productioncont{| \token{import_stmt}}
+  \productioncont{| \token{global_stmt}}
+  \productioncont{| \token{exec_stmt}}
 \end{productionlist}
 
 
@@ -112,12 +111,12 @@ objects:
   \production{target_list}
              {\token{target} ("," \token{target})* [","]}
   \production{target}
-             {\token{identifier}
-              | "(" \token{target_list} ")"
-              | "[" \token{target_list} "]"
-              | \token{attributeref}
-              | \token{subscription}
-              | \token{slicing}}
+             {\token{identifier}}
+  \productioncont{| "(" \token{target_list} ")"}
+  \productioncont{| "[" \token{target_list} "]"}
+  \productioncont{| \token{attributeref}}
+  \productioncont{| \token{subscription}}
+  \productioncont{| \token{slicing}}
 \end{productionlist}
 
 (See section~\ref{primaries} for the syntax definitions for the last
@@ -264,7 +263,7 @@ print x
 \end{verbatim}
 
 
-\subsection{Augmented Assignment statements \label{augassign}}
+\subsection{Augmented assignment statements \label{augassign}}
 
 Augmented assignment is the combination, in a single statement, of a binary
 operation and an assignment statement:
@@ -275,15 +274,8 @@ operation and an assignment statement:
   \production{augmented_assignment_stmt}
              {\token{target} \token{augop} \token{expression_list}}
   \production{augop}
-             {"+=" | "-=" | "*=" | "/=" | "\%=" | "**="
-              | ">>=" | "<<=" | "\&=" | "\textasciicircum=" | "|="}
-  \production{target}
-             {\token{identifier}
-              | "(" \token{target_list} ")"
-              | "[" \token{target_list} "]"
-              | \token{attributeref}
-              | \token{subscription}
-              | \token{slicing}}
+             {"+=" | "-=" | "*=" | "/=" | "\%=" | "**="}
+  \productioncont{| ">>=" | "<<=" | "\&=" | "\textasciicircum=" | "|="}
 \end{productionlist}
 
 (See section~\ref{primaries} for the syntax definitions for the last
@@ -364,9 +356,9 @@ right type (but even this is determined by the sliced object).
 
 \begin{productionlist}
   \production{print_stmt}
-             {"print" ( \optional{\token{expression} ("," \token{expression})* \optional{","}}
-                      | ">\code{>}" \token{expression}
-                        \optional{("," \token{expression})+ \optional{","}})}
+             {"print" ( \optional{\token{expression} ("," \token{expression})* \optional{","}}}
+  \productioncont{| ">\code{>}" \token{expression}
+                  \optional{("," \token{expression})+ \optional{","}} )}
 \end{productionlist}
 
 \keyword{print} evaluates each expression in turn and writes the
@@ -600,11 +592,11 @@ It continues with the next cycle of the nearest enclosing loop.
 \begin{productionlist}
   \production{import_stmt}
              {"import" \token{module} ["as" \token{name}]
-                ( "," \token{module} ["as" \token{name}] )*
-              | "from" \token{module} "import" \token{identifier}
-                ["as" \token{name}]
-                ( "," \token{identifier} ["as" \token{name}] )*
-              | "from" \token{module} "import" "*"}
+                ( "," \token{module} ["as" \token{name}] )*}
+  \productioncont{| "from" \token{module} "import" \token{identifier}
+                    ["as" \token{name}]}
+  \productioncont{  ( "," \token{identifier} ["as" \token{name}] )*}
+  \productioncont{| "from" \token{module} "import" "*"}
   \production{module}
              {(\token{identifier} ".")* \token{identifier}}
 \end{productionlist}

Doc/ref/ref7.tex

@@ -42,8 +42,12 @@ Summarizing:
 
 \begin{productionlist}
   \production{compound_stmt}
-             {\token{if_stmt} | \token{while_stmt} | \token{for_stmt}
-              | \token{try_stmt} | \token{funcdef} | \token{classdef}}
+             {\token{if_stmt}}
+  \productioncont{| \token{while_stmt}}
+  \productioncont{| \token{for_stmt}}
+  \productioncont{| \token{try_stmt}}
+  \productioncont{| \token{funcdef}}
+  \productioncont{| \token{classdef}}
   \production{suite}
              {\token{stmt_list} NEWLINE
               | NEWLINE INDENT \token{statement}+ DEDENT}
@@ -73,9 +77,9 @@ The \keyword{if} statement is used for conditional execution:
 
 \begin{productionlist}
   \production{if_stmt}
-             {"if" \token{expression} ":" \token{suite}
-              ( "elif" \token{expression} ":" \token{suite} )*
-              ["else" ":" \token{suite}]}
+             {"if" \token{expression} ":" \token{suite}}
+  \productioncont{( "elif" \token{expression} ":" \token{suite} )*}
+  \productioncont{["else" ":" \token{suite}]}
 \end{productionlist}
 
 It selects exactly one of the suites by evaluating the expressions one
@@ -97,8 +101,8 @@ as an expression is true:
 
 \begin{productionlist}
   \production{while_stmt}
-             {"while" \token{expression} ":" \token{suite}
-               ["else" ":" \token{suite}]}
+             {"while" \token{expression} ":" \token{suite}}
+  \productioncont{["else" ":" \token{suite}]}
 \end{productionlist}
 
 This repeatedly tests the expression and, if it is true, executes the
@@ -126,8 +130,8 @@ sequence (such as a string, tuple or list) or other iterable object:
 \begin{productionlist}
   \production{for_stmt}
              {"for" \token{target_list} "in" \token{expression_list}
-              ":" \token{suite}
-              ["else" ":" \token{suite}]}
+              ":" \token{suite}}
+  \productioncont{["else" ":" \token{suite}]}
 \end{productionlist}
 
 The expression list is evaluated once; it should yield a sequence.  The
@@ -192,10 +196,10 @@ code for a group of statements:
   \production{try_stmt}
              {\token{try_exc_stmt} | \token{try_fin_stmt}}
   \production{try_exc_stmt}
-             {"try" ":" \token{suite}
-              ("except" [\token{expression} ["," \token{target}]] ":"
-              \token{suite})+
-              ["else" ":" \token{suite}]}
+             {"try" ":" \token{suite}}
+  \productioncont{("except" [\token{expression}
+                             ["," \token{target}]] ":" \token{suite})+}
+  \productioncont{["else" ":" \token{suite}]}
   \production{try_fin_stmt}
              {"try" ":" \token{suite}
               "finally" ":" \token{suite}}
@@ -310,10 +314,10 @@ section \ref{types}):
              {"def" \token{funcname} "(" [\token{parameter_list}] ")"
               ":" \token{suite}}
   \production{parameter_list}
-             {(\token{defparameter} ",")*
-              ("*" \token{identifier} [, "**" \token{identifier}]
-               | "**" \token{identifier}
-               | \token{defparameter} [","])}
+             {(\token{defparameter} ",")*}
+  \productioncont{("*" \token{identifier} [, "**" \token{identifier}]}
+  \productioncont{| "**" \token{identifier}
+                  | \token{defparameter} [","])}
   \production{defparameter}
              {\token{parameter} ["=" \token{expression}]}
   \production{sublist}

Doc/ref/refa1.tex

@@ -20,13 +20,13 @@ versions of Python that introduce incompatible changes to the
 language.  It allows use of the new features on a per-module basis
 before the release in which the feature becomes standard.
 
-\begin{verbatim}
-future_statement: "from" "__future__" "import" feature ["as" name]
-                 ("," feature ["as" name])*
-
-feature: identifier
-name: identifier
-\end{verbatim}
+\begin{productionlist}[*]
+  \production{future_statement}
+             {"from" "__future__" "import" feature ["as" name]}
+  \productioncont{("," feature ["as" name])*}
+  \production{feature}{identifier}
+  \production{name}{identifier}
+\end{productionlist}
 
 A future statement must appear near the top of the module.  The only
 lines that can appear before a future statement are:

Doc/texinputs/python.sty

@@ -53,10 +53,10 @@
   % the same for each link, and avoids having lots of garbage all over 
   % this style file.
   \newcommand{\py@linkToName}[2]{%
-    \pdfannotlink attr{/Border [0 0 0]} goto name{#1}%
+    \pdfstartlink attr{/Border [0 0 0]} goto name{#1}%
       \py@LinkColor#2\py@NormalColor%
     \pdfendlink%
-  }    
+  }
   % Compute the padded page number separately since we end up with a pair of
   % \relax tokens; this gets the right string computed and works.
   \renewcommand{\contentsline}[3]{%
@@ -788,7 +788,7 @@
 % but only if we actually used hyperref:
 \ifpdf
   \newcommand{\url}[1]{{%
-    \pdfannotlink attr{/Border [0 0 0]} user{/S /URI /URI (#1)}%
+    \pdfstartlink attr{/Border [0 0 0]} user{/S /URI /URI (#1)}%
     \py@LinkColor%                              color of the link text
     \small\sf #1%
     \py@NormalColor%                    Turn it back off; these are declarative
@@ -864,9 +864,9 @@
 % \ulink{link text}{URL}
 \ifpdf
   % The \noindent here is a hack -- we're forcing pdfTeX into
-  % horizontal mode since \pdfannotlink requires that.
+  % horizontal mode since \pdfstartlink requires that.
   \newcommand{\ulink}[2]{\noindent{%
-    \pdfannotlink attr{/Border [0 0 0]} user{/S /URI /URI (#2)}%
+    \pdfstartlink attr{/Border [0 0 0]} user{/S /URI /URI (#2)}%
     \py@LinkColor%                              color of the link text
     #1%
     \py@NormalColor%                    Turn it back off; these are declarative
@@ -899,17 +899,7 @@
 \newenvironment{productionlist}[1][\py@badkey]{
   \def\optional##1{{\Large[}##1{\Large]}}
   \def\production##1##2{\code{##1}&::=&\code{##2}\\}
-  \def\orgroup##1{{\def\oritem{\textbar\ }##1}}
-  \def\orgroup*##1{{
-      \def\oritem{\\ \textbar&}
-      % This uses math mode's ``negative thin space'' to avoid a weird
-      % indentation that I've not been able to figure out, but
-      % probably relates to nesting tabular environments.
-      $\!\!\!\!\!\!\!\!\!\!$%
-      \begin{tabular}[t]{ll}
-        \ & ##1
-      \end{tabular}
-    }}
+  \def\productioncont##1{& &\code{##1}\\}
   \def\token##1{##1}
   \let\grammartoken=\token
   \parindent=2em