% Copyright 2012-2024, Alexander Shibakov
% This file is part of SPLinT
%
% SPLinT is free software: you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation, either version 3 of the License, or
% (at your option) any later version.
%
% SPLinT is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
% GNU General Public License for more details.
%
% You should have received a copy of the GNU General Public License
% along with SPLinT.  If not, see <http://www.gnu.org/licenses/>.

% finish parser initialization

\let\yylexreturn\yylexreturnregular

\genericparser 
    name: main, 
    ptables: cweb/gyytab.tex, 
    ltables: cweb/ltab.tex, 
    tokens: bo.tok, 
    asetup: {}, 
    dsetup: \newlexerstateextra\newparserstateextra, 
    rsetup: {},
    optimization: \optimizeall;%

% prologue parser

\genericparser 
    name: prologue, 
    ptables: cweb/dyytab.tex, 
    ltables: cweb/ltab.tex, 
    tokens: bo.tok, 
    asetup: {}, 
    dsetup: \newlexerstateextra\newparserstateextra, 
    rsetup: {},
    optimization: \optimizeall;%

% flex parser

\genericparser 
    name: flex, 
    ptables: cweb/fiptab.tex, 
    ltables: cweb/filtab.tex, 
    tokens: fo.tok, 
    asetup: {}, 
    dsetup: {},
    rsetup: {},
    optimization: {};% optimized by the driver (--optimize-tables, --optimize-actions)

% flex regex parser

\genericparser 
    name: flexre, 
    ptables: cweb/reptab.tex, 
    ltables: cweb/filtab.tex, 
    tokens: fo.tok, 
    asetup: {}, 
    dsetup: {},
    rsetup: {},
    optimization: {};% optimized by the driver (--optimize-tables, --optimize-actions)

% flex section 2 parser

\genericparser 
    name: flextwo, 
    ptables: cweb/raptab.tex, 
    ltables: cweb/filtab.tex, 
    tokens: fo.tok, 
    asetup: {}, 
    dsetup: \newparserstateextra,
    rsetup: {},
    optimization: {};% optimized by the driver (--optimize-tables, --optimize-actions)

% flex section 1 parser

\genericparser 
    name: flexone, 
    ptables: cweb/ddptab.tex, 
    ltables: cweb/filtab.tex, 
    tokens: fo.tok, 
    asetup: {}, 
    dsetup: {},
    rsetup: {},
    optimization: {};% optimized by the driver (--optimize-tables, --optimize-actions)

\let\flexpseudonamespace\flexnamespace
\let\flexpseudorenamespace\flexrenamespace

% parser for term names: this is not really a great idea in itself but rather an
% illustration of what is possible

\genericparser 
    name: small, 
    ptables: cweb/small_tab.tex, 
    ltables: cweb/small_dfa.tex, 
    tokens: {}, 
    asetup: {}, 
    dsetup: {}, 
    rsetup: \let\returnexplicitspace\ignoreexplicitspace, % ignore spaces in names
    optimization: \optimizeall;%

\tomainparser

% stage two macros: parsing

% \flex parser stack

% section 1 parser

\def\flexoneparserinit{%
    \yylessusedtrue
    \floption@sensetrue
}

\def\flexoneparserdatainit{%
    \table{}%
}

% regular expression parser

\def\flexreparserinit{%
    \yyBEGIN{SECT2}%
    \flin@ruletrue
    \yylessusedtrue
}

\def\flexreparserdatainit{%
    \table{}%
}

% section 2 parser

\def\flexparserinit{%
    \yyBEGIN{SECT2}%
    \def\flbracelevel{0}%
    \yylessusedtrue
}

\def\flexparserdatainit{%
    \table{}%
    \let\secttwoprefix\empty
}

% output parsed tables to a file

\long\def\saveoutputcode#1#2#3{% #1 is the output code (will be expanded by \write)
                                  % #2 is the output stream
                                  % #3 is the preamble (will be expanded by \write)
    \newlinechar=`^^J%
    \immediate\write#2{%
        \harmlesscomment \parsernamespace::parsed table #3:^^J#1^^J^^J%
        \harmlesscomment \parsernamespace::stashed stream:^^J\yystash^^J^^J% TODO: this just displays the name for now;
        \harmlesscomment \parsernamespace::format stream:^^J\yyformat^^J^^J% TODO: this just displays the name for now;
    }%
}

% display parsed tables and streams

\def\displayparsedoutput#1{%
    \ifchecktable 
        \ferrmessage{parsed table: \the#1^^J^^J%
                     stashed stream: \yystash^^J^^J% TODO: this just displays the name for now;
                     format stream: \yyformat}% TODO: this just displays the name for now;
    \fi
}

% display raw input

\def\displayrawtable{%
    \ifsaveparseoutput
        {\newlinechar=`^^J\immediate\write\exampletable{^^J\harmlesscomment 
         table before parsing:^^J\the\Binputtoks}}%
    \fi
    \ifchecktable 
        \ferrmessage{table before parsing: \the\Binputtoks}%
    \fi
}

\def\displayorsavefulloutput#1#2{%
    \ifchecktable 
        \ferrmessage{table after processing: \the#1}%
    \fi
    \ifsaveparseoutput
        {\newlinechar=`^^J\immediate\write#2{^^J\harmlesscomment
         processed table:^^J\the#1}}%
    \fi
}

% the macros below make sure that (almost) none of the command sequences produced by the parser are
% expandable; this makes debugging easier

\def\saveflextable#1{{%
    \hidecslist\cwebstreamchars
    \restorecslist{flexparser-debug}\yyflunion
    \expandafter\saveoutputcode\expandafter{\the\table}\exampletable{#1}%
}}

\def\savebisontable#1{{%
    \hidecslist\cwebstreamchars
    \restorecslist{parser-debug}\yyunion
    \expandafter\saveoutputcode\expandafter{\the\table}\exampletable{#1}%
}}

% stage two parsing macros

\def\preparsebisongrammar{%
    \let\postparse\postparsebisongrammar
    \tomainparser
    \displayrawtable % do this after the parse namespaces are setup
    \basicparserinit
    \bisonparserinit
    \bisonparserdatainit
    \yyparse
}

\def\preparsebisonprologue{%
    \let\postparse\postparsebisonprologue
    \toprologueparser
    \displayrawtable % do this after the namespaces are setup
    \basicparserinit
    \bisonparserinit
    \bisonparserdatainit
    \yyparse
}

\def\preparseflexone{%
    \let\postparse\postparseflexone
    \toflexoneparser
    \displayrawtable % do this after the namespaces are setup
    \basicparserinit
    \flexoneparserinit
    \flexoneparserdatainit
    \yyparse
}

\def\preparseflextwo{%
    \let\postparse\postparseflextwo
    \toflextwoparser
    \displayrawtable % do this after the namespaces are setup
    \basicparserinit
    \flexparserinit
    \flexparserdatainit
    \yyparse
}

\def\preparsefallback#1{%
    \let\postparse\relax
    \message{#1}%
}

% stage three, postprocessing and typesetting

\newif\ifparseverbose

\def\postparsegeneric#1{%
    \ifyyparsefail
        \yybreak{%
            \ifparseverbose\ferrmessage{#1 parsing failed.}\fi
            \parserreset
        }%
    \else
        \yybreak{%
            \ifparseverbose\ferrmessage{#1 parsing successful.}\fi
            \ifsaveparseoutput\saveparsedtable{#1}\fi
            \finishlist\yystash
            \finishlist\yyformat
            \typesetparsedtables
        }%
    \yycontinue
}

\def\parserreset#1\par{%
    \yyparsefailfalse % in case the next pass is a \relax
    \let\postparse\empty % ...
    \expandafter\skiptolsection\the\Binputtoks\par% start the next parsing pass, skip \6\hbox{} trash
}

\def\postparsebisongrammar{%
    \let\saveparsedtable\savebisontable
    \let\typesetparsedtables\typesetalltables
    \postparsegeneric{(grammar)}%
}

\def\postparsebisonprologue{%
    \let\saveparsedtable\savebisontable
    \let\typesetparsedtables\typesetalltables
    \postparsegeneric{(prologue)}%
}

\def\postparseflexone{%
    \let\saveparsedtable\saveflextable
    \let\typesetparsedtables\typesetfsonetables
    \postparsegeneric{(section 1)}%
}

\def\postparseflextwo{%
    \let\saveparsedtable\saveflextable
    \let\typesetparsedtables\typesetfstwotables
    \postparsegeneric{(section 2)}%
}

\fillpstack{}{%
    \preparsebisongrammar
    \preparsebisonprologue
    {\preparsefallback{**}}%
    \relax % this \relax is necessary so that the braces above 
           % are not stripped by \poppstack
}

\fillpstack{b}{%
    \preparsebisongrammar
    \preparsebisonprologue
    \preparseflexone
    {\preparsefallback{..}}%
    \relax % this \relax is necessary so that the braces above 
           % are not stripped by \poppstack
}

\fillpstack{fs1}{%
    \preparseflexone
    \preparseflextwo
    {\preparsefallback{==}}%
    \relax % this \relax is necessary so that the braces above 
           % are not stripped by \poppstack
}

\fillpstack{fs2}{%
    \preparseflextwo
    {\preparsefallback{--}}%
    \relax % this \relax is necessary so that the braces above 
           % are not stripped by \poppstack
}

\fillpstack{t}{%
    \relax
}

% stage 3.5 macros: typesetting

\def\tlskip{\z@}
\def\tfskip{\parindent}

\newif\ifchecktrailingstash

\def\typesetalltables{%
    \begingroup
        \displayparsedoutput\table
        \extractprodtableinfo
        \setprodtable
        \executeast{\postoks{}\pushothertables}%
        \parindent1em
        \checkforpropertable{\executelist\aststream}%
        \tabskip\tfskip
        \ruletableset
        {\edef\next{\write\auxstream{\nx\nx\nx\lodimens{\the\gaglue}{}{}{}\harmlesscomment}}\next}%
        \finishastoutput % debugging and stash cleanup
    \expandafter % export the value of the alignment glue
    \endgroup
    \expandafter\gaglue\the\gaglue\relax
}

\def\typesetfstwotables{%
    \begingroup
        \displayparsedoutput\table
        \extractregextableinfo % preprocess the AST for section 2 
        \setregextable % activate AST control sequences for section 2
        \executeast{}% execute AST (the output will be collected in the \aststream list)
        \regextableset % typeset the contents
        \finishastoutput % debugging and stash cleanup
    \endgroup
}

\def\typesetfsonetables{%
    \begingroup
        \displayparsedoutput\table
        \extractregextableinfo    
        \setregexdeftable
        \executeast{}% execute AST (the output will be collected in the \aststream list)
        \regexdeftableset
        \finishastoutput % debugging and stash cleanup
    \endgroup
}

\def\executeast#1{% #1 is an optional postprocessing action
                  % execute AST (the output will be collected in the \aststream list)
    \initlist\aststream
    \the\table\relax
    #1%
    \finishlist\aststream
}

\def\finishastoutput{%
    % possibly output debugging information,
    % clean up and typeset any remaining stash
    \displayorsavefulloutput\table\exampletable
    \ifchecktrailingstash
        \ferrmessage{remaining stash: \yystash}% TODO: this just displays the name for now;
    \fi
    \consumefulllist\yystash\to\toksa
    \cleanstash\stripstash\checkforccode
    \ifchecktrailingstash
        \ferrmessage{stash after cleaning: \the\toksa}%
    \fi
    \ifnum\wd0>\z@
    %\ifchecktable
    %    \showboxdepth=1000 \showboxbreadth=1000 \showbox0
    %\fi
    % currently testing for nontrivial leftover stash involves packaging the stash material
    % into a \vbox; as a result, the stash containing ${}{}$\hbox{} will have a nonzero length
    % which is why the test below is necessary
        \ifnum\ht0>\z@
            \indent\boxstash
        \fi
    \fi
}

\let\extractprodtableinfo\extractprodrefs % record the rules to be used later as a reference
\let\extractregextableinfo\empty % we do not preprocess the regex table

% setting the rule table: cross-section alignment and other effects are applied here;
% in order to produce the proper line skips before and after \unvbox, the rules followed by
% \TeX\ while adding an \halign to a vertical list have to be reproduced explicitly

\newdimen\gaglue % the width of the action box of the last alignment

\def\ruletableset{%
    \par
    \vskip-\baselineskip
    \ifx\gatoks\relax
    \else
        \expandafter\appendtolist\expandafter\aststream\expandafter{\gatoks}%
        \finishlist\aststream
    \fi
    \setbox0\vbox\expandafter{\expandafter
        \null\expandafter\prevdepth\the\prevdepth
        \halign \ifpropertable to \hsize \fi
            {\hbox to 2em{##\/$\,${\rm:}\hss}\hfil\tabskip\z@&\setallterms{##}&##\hfil\tabskip0pt plus1fil&%
                \toksa{}##\makestashbox\hfil\tabskip\tlskip\cr
                \executelist\aststream
            }
            \expandafter
    }\expandafter
    \unvcopy\expandafter0\expandafter
    \prevdepth\the\prevdepth\relax
    \setbox\z@=\vbox{\unvbox\z@ \setbox\z@=\lastbox % set the alignment dimension 
        \hbox{\unhbox\z@ \unskip\setbox\z@=\lastbox\expandafter}\expandafter}\expandafter\gaglue\the\wd\z@
}

\def\setallterms#1{\setbox\z@=\hbox{\it#1}\ifsquashterms\hbox to0pt{\unhbox\z@\hss}\else\unhbox\z@\fi\hfil}

% typesetting the scanner automaton rules

\def\regextableset{%
    \par
    \vskip-\baselineskip
    \setbox0 \vbox\expandafter{\expandafter
        \null\expandafter\prevdepth\the\prevdepth
        \halign to\hsize
            {##\hfil\tabskip0 pt plus1fil\ &\relax\tabskip\tlskip\toksa{}##\makestashbox\hfil\cr
                \executelist\aststream
            }%
            \expandafter
    }\expandafter
    \unvbox\expandafter0\expandafter
    \prevdepth\the\prevdepth\relax
}

% typesetting named regular expression definitions

\def\regexdeftableset{%
    \par
    \vskip-\baselineskip
    \setbox0\vbox\expandafter{\expandafter
        \null\expandafter\prevdepth\the\prevdepth
        \halign to\hsize
            {\hskip\parindent##\hfil\tabskip0 pt plus1fil\ &\relax\tabskip\tlskip\tt##\hfil\cr
                \executelist\aststream
            }%
            \expandafter
    }\expandafter
    \unvbox\expandafter0\expandafter
    \prevdepth\the\prevdepth\relax
}

% recording alignment layout: the gaglue, etc.

\def\yyuniontag{\auxunionctl}
\defp\tablorunionctl{}% sequence to activate/deactivate/take other action for
                      % tablor set of macros

\defc\tablorunionctl{%
    \restorecslist{aux:global:tablors}\tablorunion
}
\savecs{aux:global:activate}\tablorunionctl

\defc\tablorunionctl{%
    \restorecslist{aux:global:tablors:neutral}\tablorunion
}
\savecs{aux:global:deactivate}\tablorunionctl

\defc\tablorunionctl{% this command should not make any page material contributions
    \message{tablors...}%
}
\savecs{aux:global:preend}\tablorunionctl

\defc\tablorunionctl{% writing a header, etc.
    \message{tablors...}%
    \toksa\expandafter{\tablorunion}%
    \immediate\write\auxstream{\harmlesscomment\the\toksa}%
}
\savecs{aux:global:prestart}\tablorunionctl

\yyuniondeclare\tablorunion{aux:global:tablors}

\defp\lodimens#1#2#3#4{}

\toyyunion{aux:global:tablors:neutral}

\defc\lodimens#1#2#3#4{}% define this to set the layout for tables on a given page

\toyyunion{aux:global:tablors}

% quick and dirty global alignment: the size of the last box (and those in between)
% can be chosen automatically after one pass and read in for the final pass;
% in the future this will be the default implementation; for now, the inelegant
% solution below works as well.

\def\setglobalalignrules#1{%
    \def\gatoks{%
        \omit\hfil&\omit\hfil&\omit\hfil&\omit\hfil\hbox to #1{\hfil}\cr
        \noalign{\vskip-\baselineskip}%
        %\noalign{\centerline{$\diamond$}}
    }%
}

\setglobalalignrules{3 in}%

% to align all the actions across the document, comment out the next line

\let\gatoks\relax

\def\checkforccode{%
    \setbox0=\vbox{\setlazyc\the\toksa}%
}

\def\setlazyc{%
    \hidecs{\1\4\5\6\8}%
}

\newif\ifpropertable

\def\checkforpropertable#1{{% checking if there is any table material
% this is a hack to avoid underfull boxes when there is no actual alignment material in 
% \halign to \hsize
     \hbadness\@M
     \let\noalign\trivialnoalign
     \let\omit\relax
     \setbox\z@=\vbox{
         \halign
             {\eatone{##}&\eatone{##}.&\eatone{##}&\eatone{##}\cr
                 #1%
             }%
     }%
     \ifnum\wd\z@>\z@
         \aftergroup\propertabletrue
     \else
         \aftergroup\propertablefalse
     \fi
}}

\long\def\trivialnoalign#1{}

% macros for processing \Cee\ mode material

\newif\ifchecktrim

%\long\def\buildstash#1{\toksa\expandafter{\the\toksa#1}} % = stashed

\def\cleanstash{%
  \ifchecktrim\ferrmessage{collected stash: \the\toksa}\fi
  \expandafter\cleanst@sh\the\toksa\packagebox}

\def\cleanst@sh{\let\6\testsbox\setbox0=\vbox\bgroup}

\def\testsbox{%
  \ifmmode
      \let\next\relax
  \else
      \egroup
      \ifnum\wd\z@>\z@
          \ifnum\ht\z@>\z@
              \let\next\scoopupstash
          \else
              \let\next\rebuildstash
          \fi
      \else
          \let\next\rebuildstash
      \fi
  \fi
  \next
}

\long\def\rebuildstash#1\packagebox{\toksa{#1}\cleanstash}

\long\def\scoopupstash#1\packagebox{}

\def\packagebox{\egroup\ifnum\wd0>\z@\else\toksa{}\fi}

\def\stripstash{%
  \ifchecktrim\ferrmessage{before trimming: \the\toksa}\fi
  \def\6{}\expandafter\stripst@sh\expandafter\ignorespaces\the\toksa\6\str@pst@sh}

\def\stripst@sh{\toksa{}\stripst@shi}

\long\def\stripst@shi#1\6{%
  \toksb{#1}\futurelet\next\str@pst@sh
}

\def\str@pst@sh{%
  \ifx\next\str@pst@sh
      \iftrailingreturn
          \striptrim
      \fi
      \concat\toksa\toksb
      \iftrailingreturn % not done yet
          \let\next\stripagain
      \else
          \let\next\eatone
      \fi
      \trailingreturnfalse
  \else
      \edef\next{\toksa{\the\toksa\the\toksb\noexpand\6}}\next
      \def\next{\stripst@shi\relax\ignorespaces}%
      \trailingreturntrue
  \fi
  \next
}

\def\stripagain#1{\stripstash}

\newif\iftrailingreturn

\def\striptrim{%
  \ifchecktrim\ferrmessage{trimming: \the\toksb}\fi
  \edef\next{\the\toksb}%
  \expandafter\striptr@m\the\toksb\relax\end
}

\def\striptr@m
        #1% \relax (from last \stripst@shi)
        #2% \ignorespaces (from last \stripst@shi)
        % intervening spaces
        #3% \relax
        #4% ?
        \end{%
%  \toksc{#3#4}\showthe\toksc
%
  \setbox\z@\vbox{
      #3#4}%
  \ifnum\wd\z@=\z@
      \expandafter\trimreturn\the\toksa\end
      \toksb{}%
  \else
      \ifnum\ht\z@=\z@
          \expandafter\trimreturn\the\toksa\end
          \toksb{}%
      \else
          \trailingreturnfalse
      \fi
  \fi
}

\def\trimreturn#1\6\end{%
  \toksa{#1}%
}

\def\boxstash{%
  \ifchecktrim\ferrmessage{stash contents: \the\toksa}\fi
  $\vtop{\activateinlinec\tabskip\z@\halign{\strut\ignorespaces##\hfil\cr\relax\the\toksa\crcr}}$}

\def\makestashbox{\cleanstash\stripstash\boxstash}

\newbox\indentbox

\def\activateinlinec{%
  \setbox\indentbox=\hbox{}%
  \def\1{\setbox\indentbox\hbox{\box\indentbox\hbox to 3em{\hfil}}}%
  \def\2{\setbox\indentbox\hbox{\box\indentbox\hbox to -3em{}}}%
  \def\4{\hbox to -3em{}}%
  \let\5\ignorespaces % so that @+ does not produce a space
  \def\6{%
      \edef\setindentbox{%
          \setbox\indentbox\hbox to\the\wd\indentbox{\noexpand\hfil}%
          \copy\indentbox\ignorespaces
      }%
      \expandafter\crcontainer\setindentbox
  }%
  \def\7{%
      \edef\setindentbox{%
          \setbox\indentbox\hbox to\the\wd\indentbox{\noexpand\hfil}%
          \copy\indentbox\ignorespaces
      }%
      \expandafter\crcontainerspread\setindentbox
  }%
  \def\8{\hskip-1em}%
  \let\$\prodterm
}

\def\crcontainer{\cr}
\def\crcontainerspread{\cr\noalign{\yskip}}

% rough version of the term typesetting

\def\prodterm{%
    \futurelet\next\analyzepterm
}

\def\analyzepterm{%
    \ifx\next\$%
        \let\next\pr@dterm
    \else
        \ifcat\noexpand\next0%
            \let\next\pr@dterm
        \else
            \ifcat\noexpand\next a%
                \def\next{\hbox{$\Upsilon$}}% TODO: look for an identifier
            \else
                \let\next\oldmathS% TODO: one more category: control sequence for cases like \$\\{identifier}
            \fi
        \fi
    \fi
    \next
}
            
\def\pr@dterm#1{%
  \ifx#1\$%
      \def\next{\hbox{$\Upsilon$}}%
  \else
      \if\noexpand#1[%
          \let\next\seeksym
      \else
          \ifnum`#1<"3A\relax
              \ifnum`#1>"2F\relax % within ['0', '9']  
                  \def\next{\seekno#1}%
              \else
                  \def\next{\oldmathS#1}%
              \fi
          \else
              \def\next{\oldmathS#1}%
          \fi
      \fi
  \fi
  \next
}%

\let\$\prodterm
\defreserved\${\prodterm}

\def\seekno{\afterassignment\printterm\tempca}%

\def\seeksym#1]{%
  \hbox{$\Upsilon\kern-1pt{}_{\def\\##1{\hbox{\sscmd\prodstyle{##1}}}\rm#1}$}}

\def\seeksym#1]{% a better version of the above
  \hbox{$\ulcorner\def\\##1{##1}\let\.\\\let\|\\\let\ous\_\let\_\relax
    \edef\next{#1}\let\_\ous
    \hbox{\expandafter\prodstyle\expandafter{\next}}\urcorner$}}

\def\printterm{\hbox{$\Upsilon\kern-1pt{}_{\number\tempca}$}}%

% typesetting examples of \bison\ productions and \flex\ input in text

\long\def\setproduction#1{%
    \def\termidxrank{5}%
    \def\headeridxrank{4}%
    \def\defidxrank{3}%
    \def\texcsidxrank{5}%
    \textproductionsetup
    \hbox{\strut}%
    \Binputtoks{\lsectionbegin{b}#1\yyeof\yyeof\endparseinput\endparse\postparse}%
    \the\Binputtoks\par% Stage two, start the parsing
}

\def\textproductionsetup{%
    \ninepoint
    \let\returnexplicitspace\splitexplicitspace
    \let\acharswitch\texcharadjust
    \let\onecharswitch\texcsadjust
    \let\extractprodtableinfo\empty % we do not preprocess the table
    \showlastactionfalse
    \let\actionfiller\empty
    \fillpstack{b}{%
        \preparsebisongrammar
        \preparsebisonprologue
        {\preparsefallback{**}}%
        \relax % this \relax is necessary so that the braces above 
               % are not stripped by \poppstack
    }%
}

\def\splitexplicitspace{%
    \yyinput\{\}% 
}

\def\texcharadjust{
      `{%
          \yybyte{|}%
          \expandafter\yycp@\expandafter`\the\yybyte\relax
          \mkpurebyte
          \yyreturn
      }%
     _{%
          \yybyte{\_}%
          \expandafter\yycp@\expandafter`\the\yybyte\relax
          \mkpurebyte
          \yyreturn
      }%
}

\def\texcsadjust{
      \n {%
          \yycp@=\n
          \mkpurebyte
          \yyreturn
      }
      \^^M {%
          \yyinput\{\}% 
      }
      \' {%
          \expandafter\yyinput\benignescape'%
      }
      \\ {%
          \expandafter\expandafter\expandafter\yyinput\expandafter\benignescape\benignescape%
      }
}%

% Note that using \setspecialcharsfrom on the switches above is unnecessary (and, indeed, an error) 
% since the characters need to be read as is, with the category codes and all

% production typesetting

\def\beginprod{%
    \par
    \begingroup
        \b@ginprod
}

\long\def\b@ginprod#1\endprod{%
        \setproduction{#1}%
    \expandafter
    \endgroup
    \expandafter\gaglue\the\gaglue\relax % export the alignment width
}

\def\beginmprod{%
    $$
    \vbox\bgroup
        \def\checkforpropertable##1{\propertablefalse} % so that the table is set to its natural width
        \b@ginmprod
}

\long\def\b@ginmprod#1\endmprod{%
        \setproduction{#1}%
    \egroup
    $$%
}

% centering productions: works for rule listing only (no token declarations, etc.)

\def\begincprod#1\endcprod{{\def\tlskip{0 pt plus1fill}\let\tfskip\tlskip\beginprod#1\endprod}}

% flex examples typesetting

\long\def\setflex#1{%
    \def\fstatedefidxrank{3}%
    \def\fstateidxrank{4}%
    \def\fregexidxrank{5}%
    \textflexsetup
    \hbox{\strut}%    
    \Binputtoks{\lsectionbegin{fs1}#1\yyeof\yyeof\endparseinput\endparse\postparse}%
    \the\Binputtoks\par% Stage two, start the parsing, the \par is expected by the \parserreset
}

\def\textflexsetup{%
    \ninepoint
    %\let\returnexplicitspace\splitexplicitspace
    \let\acharswitch\texcharadjust
    %\let\onecharswitch\texcsadjust
    \let\extractprodtableinfo\empty % we do not preprocess the table
    \fillpstack{fs1}{%
        \preparseflexone % TODO
        \preparseflextwo
        {\preparsefallback{**}}%
        \relax % this \relax is necessary so that the braces above 
               % are not stripped by \poppstack
    }%
}

\def\beginflex{%
    \par
    \begingroup
    \catcode`\^^M=12 %
    \catcode`\#=12 %
        \b@ginflex%
}

\long\def\b@ginflex#1\endflex{%
        \setflex{#1}%
    \expandafter
    \endgroup
    \expandafter\gaglue\the\gaglue\relax % export the alignment width, TODO: set \gaglue in \flex
}

\let\endcprod\endgroup
\let\endmprod\endgroup
\let\endprod\endgroup

% stringing all the manuals together (disabled for now)

%\newwrite\lastpageinfo
%\newread\testeof

%\toksa\expandafter{\fin}
%\edef\fin{\the\toksa\noexpand\savelastpagenumber}
%\def\savelastpagenumber{%
%  \openout\lastpageinfo=\jobname.lpg%
%  \write\lastpageinfo{\def\noexpand\contentspagenumber{\number\pageno}\pageno=\noexpand\contentspagenumber \advance\pageno by 1}%
%  \write\lastpageinfo{\def\noexpand\secno{\number\secno}}%
%  \closeout\lastpageinfo
%}