'\"
'\" Generated from file 'me_tcl\&.man' by tcllib/doctools with format 'nroff'
'\" Copyright (c) 2005 Andreas Kupries <andreas_kupries@users\&.sourceforge\&.net>
'\"
.TH "grammar::me::tcl" n 0\&.1 tcllib "Grammar operations and usage"
.\" The -*- nroff -*- definitions below are for supplemental macros used
.\" in Tcl/Tk manual entries.
.\"
.\" .AP type name in/out ?indent?
.\"	Start paragraph describing an argument to a library procedure.
.\"	type is type of argument (int, etc.), in/out is either "in", "out",
.\"	or "in/out" to describe whether procedure reads or modifies arg,
.\"	and indent is equivalent to second arg of .IP (shouldn't ever be
.\"	needed;  use .AS below instead)
.\"
.\" .AS ?type? ?name?
.\"	Give maximum sizes of arguments for setting tab stops.  Type and
.\"	name are examples of largest possible arguments that will be passed
.\"	to .AP later.  If args are omitted, default tab stops are used.
.\"
.\" .BS
.\"	Start box enclosure.  From here until next .BE, everything will be
.\"	enclosed in one large box.
.\"
.\" .BE
.\"	End of box enclosure.
.\"
.\" .CS
.\"	Begin code excerpt.
.\"
.\" .CE
.\"	End code excerpt.
.\"
.\" .VS ?version? ?br?
.\"	Begin vertical sidebar, for use in marking newly-changed parts
.\"	of man pages.  The first argument is ignored and used for recording
.\"	the version when the .VS was added, so that the sidebars can be
.\"	found and removed when they reach a certain age.  If another argument
.\"	is present, then a line break is forced before starting the sidebar.
.\"
.\" .VE
.\"	End of vertical sidebar.
.\"
.\" .DS
.\"	Begin an indented unfilled display.
.\"
.\" .DE
.\"	End of indented unfilled display.
.\"
.\" .SO ?manpage?
.\"	Start of list of standard options for a Tk widget. The manpage
.\"	argument defines where to look up the standard options; if
.\"	omitted, defaults to "options". The options follow on successive
.\"	lines, in three columns separated by tabs.
.\"
.\" .SE
.\"	End of list of standard options for a Tk widget.
.\"
.\" .OP cmdName dbName dbClass
.\"	Start of description of a specific option.  cmdName gives the
.\"	option's name as specified in the class command, dbName gives
.\"	the option's name in the option database, and dbClass gives
.\"	the option's class in the option database.
.\"
.\" .UL arg1 arg2
.\"	Print arg1 underlined, then print arg2 normally.
.\"
.\" .QW arg1 ?arg2?
.\"	Print arg1 in quotes, then arg2 normally (for trailing punctuation).
.\"
.\" .PQ arg1 ?arg2?
.\"	Print an open parenthesis, arg1 in quotes, then arg2 normally
.\"	(for trailing punctuation) and then a closing parenthesis.
.\"
.\"	# Set up traps and other miscellaneous stuff for Tcl/Tk man pages.
.if t .wh -1.3i ^B
.nr ^l \n(.l
.ad b
.\"	# Start an argument description
.de AP
.ie !"\\$4"" .TP \\$4
.el \{\
.   ie !"\\$2"" .TP \\n()Cu
.   el          .TP 15
.\}
.ta \\n()Au \\n()Bu
.ie !"\\$3"" \{\
\&\\$1 \\fI\\$2\\fP (\\$3)
.\".b
.\}
.el \{\
.br
.ie !"\\$2"" \{\
\&\\$1	\\fI\\$2\\fP
.\}
.el \{\
\&\\fI\\$1\\fP
.\}
.\}
..
.\"	# define tabbing values for .AP
.de AS
.nr )A 10n
.if !"\\$1"" .nr )A \\w'\\$1'u+3n
.nr )B \\n()Au+15n
.\"
.if !"\\$2"" .nr )B \\w'\\$2'u+\\n()Au+3n
.nr )C \\n()Bu+\\w'(in/out)'u+2n
..
.AS Tcl_Interp Tcl_CreateInterp in/out
.\"	# BS - start boxed text
.\"	# ^y = starting y location
.\"	# ^b = 1
.de BS
.br
.mk ^y
.nr ^b 1u
.if n .nf
.if n .ti 0
.if n \l'\\n(.lu\(ul'
.if n .fi
..
.\"	# BE - end boxed text (draw box now)
.de BE
.nf
.ti 0
.mk ^t
.ie n \l'\\n(^lu\(ul'
.el \{\
.\"	Draw four-sided box normally, but don't draw top of
.\"	box if the box started on an earlier page.
.ie !\\n(^b-1 \{\
\h'-1.5n'\L'|\\n(^yu-1v'\l'\\n(^lu+3n\(ul'\L'\\n(^tu+1v-\\n(^yu'\l'|0u-1.5n\(ul'
.\}
.el \}\
\h'-1.5n'\L'|\\n(^yu-1v'\h'\\n(^lu+3n'\L'\\n(^tu+1v-\\n(^yu'\l'|0u-1.5n\(ul'
.\}
.\}
.fi
.br
.nr ^b 0
..
.\"	# VS - start vertical sidebar
.\"	# ^Y = starting y location
.\"	# ^v = 1 (for troff;  for nroff this doesn't matter)
.de VS
.if !"\\$2"" .br
.mk ^Y
.ie n 'mc \s12\(br\s0
.el .nr ^v 1u
..
.\"	# VE - end of vertical sidebar
.de VE
.ie n 'mc
.el \{\
.ev 2
.nf
.ti 0
.mk ^t
\h'|\\n(^lu+3n'\L'|\\n(^Yu-1v\(bv'\v'\\n(^tu+1v-\\n(^Yu'\h'-|\\n(^lu+3n'
.sp -1
.fi
.ev
.\}
.nr ^v 0
..
.\"	# Special macro to handle page bottom:  finish off current
.\"	# box/sidebar if in box/sidebar mode, then invoked standard
.\"	# page bottom macro.
.de ^B
.ev 2
'ti 0
'nf
.mk ^t
.if \\n(^b \{\
.\"	Draw three-sided box if this is the box's first page,
.\"	draw two sides but no top otherwise.
.ie !\\n(^b-1 \h'-1.5n'\L'|\\n(^yu-1v'\l'\\n(^lu+3n\(ul'\L'\\n(^tu+1v-\\n(^yu'\h'|0u'\c
.el \h'-1.5n'\L'|\\n(^yu-1v'\h'\\n(^lu+3n'\L'\\n(^tu+1v-\\n(^yu'\h'|0u'\c
.\}
.if \\n(^v \{\
.nr ^x \\n(^tu+1v-\\n(^Yu
\kx\h'-\\nxu'\h'|\\n(^lu+3n'\ky\L'-\\n(^xu'\v'\\n(^xu'\h'|0u'\c
.\}
.bp
'fi
.ev
.if \\n(^b \{\
.mk ^y
.nr ^b 2
.\}
.if \\n(^v \{\
.mk ^Y
.\}
..
.\"	# DS - begin display
.de DS
.RS
.nf
.sp
..
.\"	# DE - end display
.de DE
.fi
.RE
.sp
..
.\"	# SO - start of list of standard options
.de SO
'ie '\\$1'' .ds So \\fBoptions\\fR
'el .ds So \\fB\\$1\\fR
.SH "STANDARD OPTIONS"
.LP
.nf
.ta 5.5c 11c
.ft B
..
.\"	# SE - end of list of standard options
.de SE
.fi
.ft R
.LP
See the \\*(So manual entry for details on the standard options.
..
.\"	# OP - start of full description for a single option
.de OP
.LP
.nf
.ta 4c
Command-Line Name:	\\fB\\$1\\fR
Database Name:	\\fB\\$2\\fR
Database Class:	\\fB\\$3\\fR
.fi
.IP
..
.\"	# CS - begin code excerpt
.de CS
.RS
.nf
.ta .25i .5i .75i 1i
..
.\"	# CE - end code excerpt
.de CE
.fi
.RE
..
.\"	# UL - underline word
.de UL
\\$1\l'|0\(ul'\\$2
..
.\"	# QW - apply quotation marks to word
.de QW
.ie '\\*(lq'"' ``\\$1''\\$2
.\"" fix emacs highlighting
.el \\*(lq\\$1\\*(rq\\$2
..
.\"	# PQ - apply parens and quotation marks to word
.de PQ
.ie '\\*(lq'"' (``\\$1''\\$2)\\$3
.\"" fix emacs highlighting
.el (\\*(lq\\$1\\*(rq\\$2)\\$3
..
.\"	# QR - quoted range
.de QR
.ie '\\*(lq'"' ``\\$1''\\-``\\$2''\\$3
.\"" fix emacs highlighting
.el \\*(lq\\$1\\*(rq\\-\\*(lq\\$2\\*(rq\\$3
..
.\"	# MT - "empty" string
.de MT
.QW ""
..
.BS
.SH NAME
grammar::me::tcl \- Virtual machine implementation I for parsing token streams
.SH SYNOPSIS
package require \fBTcl  8\&.4\fR
.sp
package require \fBgrammar::me::tcl  ?0\&.1?\fR
.sp
\fB::grammar::me::tcl\fR \fBcmd\fR \fI\&.\&.\&.\fR
.sp
\fB::grammar::me::tcl\fR \fBinit\fR \fInextcmd\fR ?\fItokmap\fR?
.sp
\fB::grammar::me::tcl\fR \fBlc\fR \fIlocation\fR
.sp
\fB::grammar::me::tcl\fR \fBtok\fR \fIfrom\fR ?\fIto\fR?
.sp
\fB::grammar::me::tcl\fR \fBtokens\fR
.sp
\fB::grammar::me::tcl\fR \fBsv\fR
.sp
\fB::grammar::me::tcl\fR \fBast\fR
.sp
\fB::grammar::me::tcl\fR \fBastall\fR
.sp
\fB::grammar::me::tcl\fR \fBctok\fR
.sp
\fB::grammar::me::tcl\fR \fBnc\fR
.sp
\fB::grammar::me::tcl\fR \fBnext\fR
.sp
\fB::grammar::me::tcl\fR \fBord\fR
.sp
\fB::grammar::me::tcl::ict_advance\fR \fImessage\fR
.sp
\fB::grammar::me::tcl::ict_match_token\fR \fItok\fR \fImessage\fR
.sp
\fB::grammar::me::tcl::ict_match_tokrange\fR \fItokbegin\fR \fItokend\fR \fImessage\fR
.sp
\fB::grammar::me::tcl::ict_match_tokclass\fR \fIcode\fR \fImessage\fR
.sp
\fB::grammar::me::tcl::inc_restore\fR \fInt\fR
.sp
\fB::grammar::me::tcl::inc_save\fR \fInt\fR \fIstartlocation\fR
.sp
\fB::grammar::me::tcl::iok_ok\fR
.sp
\fB::grammar::me::tcl::iok_fail\fR
.sp
\fB::grammar::me::tcl::iok_negate\fR
.sp
\fB::grammar::me::tcl::icl_get\fR
.sp
\fB::grammar::me::tcl::icl_rewind\fR \fIoldlocation\fR
.sp
\fB::grammar::me::tcl::ier_get\fR
.sp
\fB::grammar::me::tcl::ier_clear\fR
.sp
\fB::grammar::me::tcl::ier_nonterminal\fR \fImessage\fR \fIlocation\fR
.sp
\fB::grammar::me::tcl::ier_merge\fR \fIolderror\fR
.sp
\fB::grammar::me::tcl::isv_clear\fR
.sp
\fB::grammar::me::tcl::isv_terminal\fR
.sp
\fB::grammar::me::tcl::isv_nonterminal_leaf\fR \fInt\fR \fIstartlocation\fR
.sp
\fB::grammar::me::tcl::isv_nonterminal_range\fR \fInt\fR \fIstartlocation\fR
.sp
\fB::grammar::me::tcl::isv_nonterminal_reduce\fR \fInt\fR \fIstartlocation\fR ?\fImarker\fR?
.sp
\fB::grammar::me::tcl::ias_push\fR
.sp
\fB::grammar::me::tcl::ias_mark\fR
.sp
\fB::grammar::me::tcl::ias_pop2mark\fR \fImarker\fR
.sp
.BE
.SH DESCRIPTION
.PP
This package provides an implementation of the ME virtual machine\&.
Please go and read the document \fBgrammar::me_intro\fR first if
you do not know what a ME virtual machine is\&.
.PP
This implementation is tied very strongly to Tcl\&. All the stacks in
the machine state are handled through the Tcl stack, all control flow
is handled by Tcl commands, and the remaining machine instructions are
directly mapped to Tcl commands\&. Especially the matching of
nonterminal symbols is handled by Tcl procedures as well, essentially
extending the machine implementation with custom instructions\&.
.PP
Further on the implementation handles only a single machine which is
uninteruptible during execution and hardwired for pull operation\&. I\&.e\&.
it explicitly requests each new token through a callback, pulling them
into its state\&.
.PP
A related package is \fBgrammar::peg::interp\fR which provides a
generic interpreter / parser for parsing expression grammars (PEGs),
implemented on top of this implementation of the ME virtual machine\&.
.SH API
The commands documented in this section do not implement any of the
instructions of the ME virtual machine\&. They provide the facilities
for the initialization of the machine and the retrieval of important
information\&.
.TP
\fB::grammar::me::tcl\fR \fBcmd\fR \fI\&.\&.\&.\fR
This is an ensemble command providing access to the commands listed in
this section\&. See the methods themselves for detailed specifications\&.
.TP
\fB::grammar::me::tcl\fR \fBinit\fR \fInextcmd\fR ?\fItokmap\fR?
This command (re)initializes the machine\&. It returns the empty
string\&. This command has to be invoked before any other command of
this package\&.
.sp
The command prefix \fInextcmd\fR represents the input stream of
characters and is invoked by the machine whenever the a new character
from the stream is required\&. The instruction for handling this is
\fIict_advance\fR\&.
The callback has to return either the empty list, or a list of 4
elements containing the token, its lexeme attribute, and its location
as line number and column index, in this order\&.
The empty list is the signal that the end of the input stream has been
reached\&. The lexeme attribute is stored in the terminal cache, but
otherwise not used by the machine\&.
.sp
The optional dictionary \fItokmap\fR maps from tokens to integer
numbers\&. If present the numbers impose an order on the tokens, which
is subsequently used by \fIict_match_tokrange\fR to determine if a
token is in the specified range or not\&. If no token map is specified
the lexicographic order of th token names will be used instead\&. This
choice is especially asensible when using characters as tokens\&.
.TP
\fB::grammar::me::tcl\fR \fBlc\fR \fIlocation\fR
This command converts the location of a token given as offset in the
input stream into the associated line number and column index\&. The
result of the command is a 2-element list containing the two values,
in the order mentioned in the previous sentence\&.
This allows higher levels to convert the location information found in
the error status and the generated AST into more human readable data\&.
.sp
\fINote\fR that the command is not able to convert locations which
have not been reached by the machine yet\&. In other words, if the
machine has read 7 tokens the command is able to convert the offsets
\fB0\fR to \fB6\fR, but nothing beyond that\&. This also shows that
it is not possible to convert offsets which refer to locations before
the beginning of the stream\&.
.sp
After a call of \fBinit\fR the state used for the conversion is
cleared, making further conversions impossible until the machine has
read tokens again\&.
.TP
\fB::grammar::me::tcl\fR \fBtok\fR \fIfrom\fR ?\fIto\fR?
This command returns a Tcl list containing the part of the input
stream between the locations \fIfrom\fR and \fIto\fR (both
inclusive)\&. If \fIto\fR is not specified it will default to the value
of \fIfrom\fR\&.
.sp
Each element of the returned list is a list of four elements, the
token, its associated lexeme, line number, and column index, in this
order\&.
In other words, each element has the same structure as the result of
the \fInextcmd\fR callback given to \fB::grammar::me::tcl::init\fR
.sp
This command places the same restrictions on its location arguments as
\fB::grammar::me::tcl::lc\fR\&.
.TP
\fB::grammar::me::tcl\fR \fBtokens\fR
This command returns the number of tokens currently known to the ME
virtual machine\&.
.TP
\fB::grammar::me::tcl\fR \fBsv\fR
This command returns the current semantic value \fISV\fR stored in
the machine\&. This is an abstract syntax tree as specified in the
document \fBgrammar::me_ast\fR, section \fBAST VALUES\fR\&.
.TP
\fB::grammar::me::tcl\fR \fBast\fR
This method returns the abstract syntax tree currently at the top of
the AST stack of the ME virtual machine\&. This is an abstract syntax
tree as specified in the document \fBgrammar::me_ast\fR, section
\fBAST VALUES\fR\&.
.TP
\fB::grammar::me::tcl\fR \fBastall\fR
This method returns the whole stack of abstract syntax trees currently
known to the ME virtual machine\&. Each element of the returned list is
an abstract syntax tree as specified in the document
\fBgrammar::me_ast\fR, section \fBAST VALUES\fR\&.
The top of the stack resides at the end of the list\&.
.TP
\fB::grammar::me::tcl\fR \fBctok\fR
This method returns the current token considered by the ME virtual
machine\&.
.TP
\fB::grammar::me::tcl\fR \fBnc\fR
This method returns the contents of the nonterminal cache as a
dictionary mapping from "\fBsymbol\fR,\fBlocation\fR" to match
information\&.
.TP
\fB::grammar::me::tcl\fR \fBnext\fR
This method returns the next token callback as specified during
initialization of the ME virtual machine\&.
.TP
\fB::grammar::me::tcl\fR \fBord\fR
This method returns a dictionary containing the \fItokmap\fR specified
during initialization of the ME virtual machine\&.
\fB\fB::grammar::me::tcl::ok\fR\fR
This variable contains the current match status \fIOK\fR\&. It is
provided as variable instead of a command because that makes access to
this information faster, and the speed of access is considered very
important here as this information is used constantly to determine the
control flow\&.
.PP
.PP
.SH "MACHINE STATE"
Please go and read the document \fBgrammar::me_vm\fR first for a
specification of the basic ME virtual machine and its state\&.
.PP
This implementation manages the state described in that document,
except for the stacks minus the AST stack\&. In other words, location
stack, error stack, return stack, and ast marker stack are implicitly
managed through standard Tcl scoping, i\&.e\&. Tcl variables in
procedures, outside of this implementation\&.
.SH "MACHINE INSTRUCTIONS"
Please go and read the document \fBgrammar::me_vm\fR first for a
specification of the basic ME virtual machine and its instruction set\&.
.PP
This implementation maps all instructions to Tcl commands in the
namespace "::grammar::me::tcl", except for the stack related commands,
nonterminal symbols and control flow\&.
Here we simply list the commands and explain the differences to the
specified instructions, if there are any\&. For their semantics see the
aforementioned specification\&. The machine commands are \fInot\fR
reachable through the ensemble command \fB::grammar::me::tcl\fR\&.
.PP
.TP
\fB::grammar::me::tcl::ict_advance\fR \fImessage\fR
No changes\&.
.TP
\fB::grammar::me::tcl::ict_match_token\fR \fItok\fR \fImessage\fR
No changes\&.
.TP
\fB::grammar::me::tcl::ict_match_tokrange\fR \fItokbegin\fR \fItokend\fR \fImessage\fR
If, and only if a token map was specified during initialization then
the arguments are the numeric representations of the smallest and
largest tokens in the range\&. Otherwise they are the relevant tokens
themselves and lexicographic comparison is used\&.
.TP
\fB::grammar::me::tcl::ict_match_tokclass\fR \fIcode\fR \fImessage\fR
No changes\&.
.TP
\fB::grammar::me::tcl::inc_restore\fR \fInt\fR
Instead of taking a branchlabel the command returns a boolean value\&.
The result will be \fBtrue\fR if and only if cached information was
found\&. The caller has to perform the appropriate branching\&.
.TP
\fB::grammar::me::tcl::inc_save\fR \fInt\fR \fIstartlocation\fR
The command takes the start location as additional argument, as it is
managed on the Tcl stack, and not in the machine state\&.
.TP
\fBicf_ntcall\fR \fIbranchlabel\fR
.TP
\fBicf_ntreturn\fR
These two instructions are not mapped to commands\&. They are control
flow instructions and handled in Tcl\&.
.TP
\fB::grammar::me::tcl::iok_ok\fR
No changes\&.
.TP
\fB::grammar::me::tcl::iok_fail\fR
No changes\&.
.TP
\fB::grammar::me::tcl::iok_negate\fR
No changes\&.
.TP
\fBicf_jalways\fR \fIbranchlabel\fR
.TP
\fBicf_jok\fR \fIbranchlabel\fR
.TP
\fBicf_jfail\fR \fIbranchlabel\fR
.TP
\fBicf_halt\fR
These four instructions are not mapped to commands\&. They are control
flow instructions and handled in Tcl\&.
.TP
\fB::grammar::me::tcl::icl_get\fR
This command returns the current location \fICL\fR in the input\&.
It replaces \fIicl_push\fR\&.
.TP
\fB::grammar::me::tcl::icl_rewind\fR \fIoldlocation\fR
The command takes the location as argument as it comes from the
Tcl stack, not the machine state\&.
.TP
\fBicl_pop\fR
Not mapped, the stacks are not managed by the package\&.
.TP
\fB::grammar::me::tcl::ier_get\fR
This command returns the current error state \fIER\fR\&.
It replaces \fIier_push\fR\&.
.TP
\fB::grammar::me::tcl::ier_clear\fR
No changes\&.
.TP
\fB::grammar::me::tcl::ier_nonterminal\fR \fImessage\fR \fIlocation\fR
The command takes the location as argument as it comes from the
Tcl stack, not the machine state\&.
.TP
\fB::grammar::me::tcl::ier_merge\fR \fIolderror\fR
The command takes the second error state to merge as argument as it
comes from the Tcl stack, not the machine state\&.
.TP
\fB::grammar::me::tcl::isv_clear\fR
No changes\&.
.TP
\fB::grammar::me::tcl::isv_terminal\fR
No changes\&.
.TP
\fB::grammar::me::tcl::isv_nonterminal_leaf\fR \fInt\fR \fIstartlocation\fR
The command takes the start location as argument as it comes from the
Tcl stack, not the machine state\&.
.TP
\fB::grammar::me::tcl::isv_nonterminal_range\fR \fInt\fR \fIstartlocation\fR
The command takes the start location as argument as it comes from the
Tcl stack, not the machine state\&.
.TP
\fB::grammar::me::tcl::isv_nonterminal_reduce\fR \fInt\fR \fIstartlocation\fR ?\fImarker\fR?
The command takes start location and marker as argument as it comes
from the Tcl stack, not the machine state\&.
.TP
\fB::grammar::me::tcl::ias_push\fR
No changes\&.
.TP
\fB::grammar::me::tcl::ias_mark\fR
This command returns a marker for the current state of the AST stack
\fIAS\fR\&. The marker stack is not managed by the machine\&.
.TP
\fB::grammar::me::tcl::ias_pop2mark\fR \fImarker\fR
The command takes the marker as argument as it comes from the
Tcl stack, not the machine state\&. It replaces \fIias_mpop\fR\&.
.PP
.PP
.SH "BUGS, IDEAS, FEEDBACK"
This document, and the package it describes, will undoubtedly contain
bugs and other problems\&.
Please report such in the category \fIgrammar_me\fR of the
\fITcllib Trackers\fR [http://core\&.tcl\&.tk/tcllib/reportlist]\&.
Please also report any ideas for enhancements you may have for either
package and/or documentation\&.
.PP
When proposing code changes, please provide \fIunified diffs\fR,
i\&.e the output of \fBdiff -u\fR\&.
.PP
Note further that \fIattachments\fR are strongly preferred over
inlined patches\&. Attachments can be made by going to the \fBEdit\fR
form of the ticket immediately after its creation, and then using the
left-most button in the secondary navigation bar\&.
.SH KEYWORDS
grammar, parsing, virtual machine
.SH CATEGORY
Grammars and finite automata
.SH COPYRIGHT
.nf
Copyright (c) 2005 Andreas Kupries <andreas_kupries@users\&.sourceforge\&.net>

.fi